diff options
537 files changed, 43688 insertions, 12922 deletions
diff --git a/gcc/testsuite/go.test/test/chan/sieve2.go b/gcc/testsuite/go.test/test/chan/sieve2.go index 7f2ed91..9a7ab15 100644 --- a/gcc/testsuite/go.test/test/chan/sieve2.go +++ b/gcc/testsuite/go.test/test/chan/sieve2.go @@ -13,7 +13,6 @@ package main import ( "container/heap" "container/ring" - "container/vector" ) // Return a chan of odd numbers, starting from 5. @@ -47,13 +46,28 @@ type PeekCh struct { ch chan int } -// Heap of PeekCh, sorting by head values. -type PeekChHeap struct { - *vector.Vector -} +// Heap of PeekCh, sorting by head values, satisfies Heap interface. +type PeekChHeap []*PeekCh func (h *PeekChHeap) Less(i, j int) bool { - return h.At(i).(*PeekCh).head < h.At(j).(*PeekCh).head + return (*h)[i].head < (*h)[j].head +} + +func (h *PeekChHeap) Swap(i, j int) { + (*h)[i], (*h)[j] = (*h)[j], (*h)[i] +} + +func (h *PeekChHeap) Len() int { + return len(*h) +} + +func (h *PeekChHeap) Pop() (v interface{}) { + *h, v = (*h)[:h.Len()-1], (*h)[h.Len()-1] + return +} + +func (h *PeekChHeap) Push(v interface{}) { + *h = append(*h, v.(*PeekCh)) } // Return a channel to serve as a sending proxy to 'out'. @@ -108,26 +122,26 @@ func Sieve() chan int { // Merge channels of multiples of 'primes' into 'composites'. go func() { - h := &PeekChHeap{new(vector.Vector)} + var h PeekChHeap min := 15 for { m := multiples(<-primes) head := <-m for min < head { composites <- min - minchan := heap.Pop(h).(*PeekCh) + minchan := heap.Pop(&h).(*PeekCh) min = minchan.head minchan.head = <-minchan.ch - heap.Push(h, minchan) + heap.Push(&h, minchan) } for min == head { - minchan := heap.Pop(h).(*PeekCh) + minchan := heap.Pop(&h).(*PeekCh) min = minchan.head minchan.head = <-minchan.ch - heap.Push(h, minchan) + heap.Push(&h, minchan) } composites <- head - heap.Push(h, &PeekCh{<-m, m}) + heap.Push(&h, &PeekCh{<-m, m}) } }() diff --git a/gcc/testsuite/go.test/test/vectors.go b/gcc/testsuite/go.test/test/vectors.go deleted file mode 100644 index ed5905d..0000000 --- a/gcc/testsuite/go.test/test/vectors.go +++ /dev/null @@ -1,64 +0,0 @@ -// $G $F.go && $L $F.$A && ./$A.out - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "container/vector" - - -type S struct { - val int -} - - -func (p *S) Init(val int) *S { - p.val = val - return p -} - - -func test0() { - v := new(vector.Vector) - if v.Len() != 0 { - print("len = ", v.Len(), "\n") - panic("fail") - } -} - - -func test1() { - var a [1000]*S - for i := 0; i < len(a); i++ { - a[i] = new(S).Init(i) - } - - v := new(vector.Vector) - for i := 0; i < len(a); i++ { - v.Insert(0, a[i]) - if v.Len() != i+1 { - print("len = ", v.Len(), "\n") - panic("fail") - } - } - - for i := 0; i < v.Len(); i++ { - x := v.At(i).(*S) - if x.val != v.Len()-i-1 { - print("expected ", i, ", found ", x.val, "\n") - panic("fail") - } - } - - for v.Len() > 10 { - v.Delete(10) - } -} - - -func main() { - test0() - test1() -} diff --git a/libgo/MERGE b/libgo/MERGE index 8ad9d1e..8ba17de 100644 --- a/libgo/MERGE +++ b/libgo/MERGE @@ -1,4 +1,4 @@ -c1702f36df03 +6d7136d74b65 The first line of this file holds the Mercurial revision number of the last merge done from the master library sources. diff --git a/libgo/Makefile.am b/libgo/Makefile.am index ce0fe83..e3b33a8 100644 --- a/libgo/Makefile.am +++ b/libgo/Makefile.am @@ -107,7 +107,6 @@ toolexeclibgo_DATA = \ cmath.gox \ crypto.gox \ csv.gox \ - ebnf.gox \ exec.gox \ expvar.gox \ flag.gox \ @@ -124,7 +123,6 @@ toolexeclibgo_DATA = \ mail.gox \ mime.gox \ net.gox \ - netchan.gox \ os.gox \ patch.gox \ path.gox \ @@ -145,7 +143,6 @@ toolexeclibgo_DATA = \ template.gox \ testing.gox \ time.gox \ - try.gox \ unicode.gox \ url.gox \ utf16.gox \ @@ -173,13 +170,13 @@ toolexeclibgocontainerdir = $(toolexeclibgodir)/container toolexeclibgocontainer_DATA = \ container/heap.gox \ container/list.gox \ - container/ring.gox \ - container/vector.gox + container/ring.gox toolexeclibgocryptodir = $(toolexeclibgodir)/crypto toolexeclibgocrypto_DATA = \ crypto/aes.gox \ + crypto/bcrypt.gox \ crypto/blowfish.gox \ crypto/cast5.gox \ crypto/cipher.gox \ @@ -242,19 +239,24 @@ toolexeclibgoencoding_DATA = \ toolexeclibgoexpdir = $(toolexeclibgodir)/exp toolexeclibgoexp_DATA = \ + exp/ebnf.gox \ exp/gui.gox \ exp/norm.gox \ - exp/regexp.gox + exp/spdy.gox \ + exp/sql.gox \ + exp/ssh.gox \ + exp/terminal.gox \ + exp/types.gox toolexeclibgoexpguidir = $(toolexeclibgoexpdir)/gui toolexeclibgoexpgui_DATA = \ exp/gui/x11.gox -toolexeclibgoexpregexpdir = $(toolexeclibgoexpdir)/regexp +toolexeclibgoexpsqldir = $(toolexeclibgoexpdir)/sql -toolexeclibgoexpregexp_DATA = \ - exp/regexp/syntax.gox +toolexeclibgoexpsql_DATA = \ + exp/sql/driver.gox toolexeclibgoexptemplatedir = $(toolexeclibgoexpdir)/template @@ -270,9 +272,7 @@ toolexeclibgogo_DATA = \ go/parser.gox \ go/printer.gox \ go/scanner.gox \ - go/token.gox \ - go/typechecker.gox \ - go/types.gox + go/token.gox toolexeclibgohashdir = $(toolexeclibgodir)/hash @@ -288,13 +288,13 @@ toolexeclibgohttp_DATA = \ http/cgi.gox \ http/fcgi.gox \ http/httptest.gox \ - http/pprof.gox \ - http/spdy.gox + http/pprof.gox toolexeclibgoimagedir = $(toolexeclibgodir)/image toolexeclibgoimage_DATA = \ image/bmp.gox \ + image/color.gox \ image/draw.gox \ image/gif.gox \ image/jpeg.gox \ @@ -326,6 +326,8 @@ toolexeclibgonet_DATA = \ toolexeclibgoolddir = $(toolexeclibgodir)/old toolexeclibgoold_DATA = \ + old/netchan.gox \ + old/regexp.gox \ old/template.gox toolexeclibgoosdir = $(toolexeclibgodir)/os @@ -347,6 +349,11 @@ toolexeclibgopathdir = $(toolexeclibgodir)/path toolexeclibgopath_DATA = \ path/filepath.gox +toolexeclibgoregexpdir = $(toolexeclibgodir)/regexp + +toolexeclibgoregexp_DATA = \ + regexp/syntax.gox + toolexeclibgorpcdir = $(toolexeclibgodir)/rpc toolexeclibgorpc_DATA = \ @@ -479,6 +486,7 @@ runtime_files = \ runtime/msize.c \ runtime/proc.c \ runtime/thread.c \ + runtime/yield.c \ $(rtems_task_variable_add_file) \ chan.c \ iface.c \ @@ -559,10 +567,6 @@ go_csv_files = \ go/csv/reader.go \ go/csv/writer.go -go_ebnf_files = \ - go/ebnf/ebnf.go \ - go/ebnf/parser.go - go_exec_files = \ go/exec/exec.go \ go/exec/lp_unix.go @@ -598,6 +602,7 @@ go_html_files = \ go/html/escape.go \ go/html/node.go \ go/html/parse.go \ + go/html/render.go \ go/html/token.go go_http_files = \ @@ -605,6 +610,7 @@ go_http_files = \ go/http/client.go \ go/http/cookie.go \ go/http/dump.go \ + go/http/filetransport.go \ go/http/fs.go \ go/http/header.go \ go/http/lex.go \ @@ -619,7 +625,6 @@ go_http_files = \ go/http/transport.go go_image_files = \ - go/image/color.go \ go/image/format.go \ go/image/geom.go \ go/image/image.go \ @@ -642,6 +647,7 @@ go_log_files = \ go/log/log.go go_math_files = \ + go/math/abs.go \ go/math/acosh.go \ go/math/asin.go \ go/math/asinh.go \ @@ -652,15 +658,13 @@ go_math_files = \ go/math/cbrt.go \ go/math/const.go \ go/math/copysign.go \ + go/math/dim.go \ go/math/erf.go \ go/math/exp.go \ go/math/exp_port.go \ go/math/exp2.go \ go/math/expm1.go \ - go/math/fabs.go \ - go/math/fdim.go \ go/math/floor.go \ - go/math/fmod.go \ go/math/frexp.go \ go/math/gamma.go \ go/math/hypot.go \ @@ -674,6 +678,7 @@ go_math_files = \ go/math/log1p.go \ go/math/log10.go \ go/math/logb.go \ + go/math/mod.go \ go/math/modf.go \ go/math/nextafter.go \ go/math/pow.go \ @@ -777,11 +782,6 @@ go_net_files = \ go/net/unixsock.go \ go/net/unixsock_posix.go -go_netchan_files = \ - go/netchan/common.go \ - go/netchan/export.go \ - go/netchan/import.go - if LIBGO_IS_SOLARIS if LIBGO_IS_386 go_os_dir_file = go/os/dir_largefile.go @@ -865,6 +865,7 @@ go_reflect_files = \ go/reflect/value.go go_regexp_files = \ + go/regexp/exec.go \ go/regexp/regexp.go go_rpc_files = \ @@ -915,6 +916,7 @@ go_strconv_files = \ go_strings_files = \ go/strings/reader.go \ + go/strings/replace.go \ go/strings/strings.go go_sync_files = \ @@ -953,21 +955,19 @@ go_template_files = \ go_testing_files = \ go/testing/benchmark.go \ + go/testing/example.go \ go/testing/testing.go go_time_files = \ go/time/format.go \ go/time/sleep.go \ go/time/sys.go \ - go/time/sys_posix.go \ + go/time/sys_unix.go \ go/time/tick.go \ go/time/time.go \ go/time/zoneinfo_posix.go \ go/time/zoneinfo_unix.go -go_try_files = \ - go/try/try.go - go_unicode_files = \ go/unicode/casetables.go \ go/unicode/digit.go \ @@ -987,6 +987,8 @@ go_utf8_files = \ go_websocket_files = \ go/websocket/client.go \ + go/websocket/hixie.go \ + go/websocket/hybi.go \ go/websocket/server.go \ go/websocket/websocket.go @@ -1041,16 +1043,13 @@ go_container_list_files = \ go_container_ring_files = \ go/container/ring/ring.go -go_container_vector_files = \ - go/container/vector/defs.go \ - go/container/vector/intvector.go \ - go/container/vector/stringvector.go \ - go/container/vector/vector.go - go_crypto_aes_files = \ go/crypto/aes/block.go \ go/crypto/aes/cipher.go \ go/crypto/aes/const.go +go_crypto_bcrypt_files = \ + go/crypto/bcrypt/base64.go \ + go/crypto/bcrypt/bcrypt.go go_crypto_blowfish_files = \ go/crypto/blowfish/block.go \ go/crypto/blowfish/const.go \ @@ -1123,11 +1122,13 @@ go_crypto_tls_files = \ go/crypto/tls/handshake_server.go \ go/crypto/tls/key_agreement.go \ go/crypto/tls/prf.go \ + go/crypto/tls/root_unix.go \ go/crypto/tls/tls.go go_crypto_twofish_files = \ go/crypto/twofish/twofish.go go_crypto_x509_files = \ go/crypto/x509/cert_pool.go \ + go/crypto/x509/pkcs1.go \ go/crypto/x509/verify.go \ go/crypto/x509/x509.go go_crypto_xtea_files = \ @@ -1187,7 +1188,8 @@ go_encoding_base32_files = \ go_encoding_base64_files = \ go/encoding/base64/base64.go go_encoding_binary_files = \ - go/encoding/binary/binary.go + go/encoding/binary/binary.go \ + go/encoding/binary/varint.go go_encoding_git85_files = \ go/encoding/git85/git.go go_encoding_hex_files = \ @@ -1195,33 +1197,66 @@ go_encoding_hex_files = \ go_encoding_pem_files = \ go/encoding/pem/pem.go +go_exp_ebnf_files = \ + go/exp/ebnf/ebnf.go \ + go/exp/ebnf/parser.go go_exp_gui_files = \ go/exp/gui/gui.go go_exp_norm_files = \ go/exp/norm/composition.go \ go/exp/norm/forminfo.go \ + go/exp/norm/input.go \ go/exp/norm/normalize.go \ + go/exp/norm/readwriter.go \ go/exp/norm/tables.go \ go/exp/norm/trie.go -go_exp_regexp_files = \ - go/exp/regexp/exec.go \ - go/exp/regexp/regexp.go +go_exp_spdy_files = \ + go/exp/spdy/read.go \ + go/exp/spdy/types.go \ + go/exp/spdy/write.go +go_exp_sql_files = \ + go/exp/sql/convert.go \ + go/exp/sql/sql.go +go_exp_ssh_files = \ + go/exp/ssh/channel.go \ + go/exp/ssh/common.go \ + go/exp/ssh/doc.go \ + go/exp/ssh/messages.go \ + go/exp/ssh/server.go \ + go/exp/ssh/server_shell.go \ + go/exp/ssh/transport.go +go_exp_terminal_files = \ + go/exp/terminal/shell.go \ + go/exp/terminal/terminal.go +go_exp_types_files = \ + go/exp/types/check.go \ + go/exp/types/const.go \ + go/exp/types/exportdata.go \ + go/exp/types/gcimporter.go \ + go/exp/types/types.go \ + go/exp/types/universe.go go_exp_gui_x11_files = \ go/exp/gui/x11/auth.go \ go/exp/gui/x11/conn.go +go_exp_sql_driver_files = \ + go/exp/sql/driver/driver.go \ + go/exp/sql/driver/types.go + go_exp_template_html_files = \ + go/exp/template/html/attr.go \ + go/exp/template/html/clone.go \ + go/exp/template/html/content.go \ go/exp/template/html/context.go \ - go/exp/template/html/escape.go - -go_exp_regexp_syntax_files = \ - go/exp/regexp/syntax/compile.go \ - go/exp/regexp/syntax/parse.go \ - go/exp/regexp/syntax/perl_groups.go \ - go/exp/regexp/syntax/prog.go \ - go/exp/regexp/syntax/regexp.go \ - go/exp/regexp/syntax/simplify.go + go/exp/template/html/css.go \ + go/exp/template/html/doc.go \ + go/exp/template/html/error.go \ + go/exp/template/html/escape.go \ + go/exp/template/html/html.go \ + go/exp/template/html/js.go \ + go/exp/template/html/transition.go \ + go/exp/template/html/url.go go_go_ast_files = \ go/go/ast/ast.go \ @@ -1237,7 +1272,8 @@ go_go_build_files = \ syslist.go go_go_doc_files = \ go/go/doc/comment.go \ - go/go/doc/doc.go + go/go/doc/doc.go \ + go/go/doc/example.go go_go_parser_files = \ go/go/parser/interface.go \ go/go/parser/parser.go @@ -1249,19 +1285,8 @@ go_go_scanner_files = \ go/go/scanner/scanner.go go_go_token_files = \ go/go/token/position.go \ + go/go/token/serialize.go \ go/go/token/token.go -go_go_typechecker_files = \ - go/go/typechecker/scope.go \ - go/go/typechecker/type.go \ - go/go/typechecker/typechecker.go \ - go/go/typechecker/universe.go -go_go_types_files = \ - go/go/types/check.go \ - go/go/types/const.go \ - go/go/types/exportdata.go \ - go/go/types/gcimporter.go \ - go/go/types/types.go \ - go/go/types/universe.go go_hash_adler32_files = \ go/hash/adler32/adler32.go @@ -1284,14 +1309,13 @@ go_http_httptest_files = \ go/http/httptest/server.go go_http_pprof_files = \ go/http/pprof/pprof.go -go_http_spdy_files = \ - go/http/spdy/read.go \ - go/http/spdy/types.go \ - go/http/spdy/write.go go_image_bmp_files = \ go/image/bmp/reader.go +go_image_color_files = \ + go/image/color/color.go + go_image_draw_files = \ go/image/draw/draw.go @@ -1311,6 +1335,7 @@ go_image_png_files = \ go_image_tiff_files = \ go/image/tiff/buffer.go \ + go/image/tiff/compress.go \ go/image/tiff/consts.go \ go/image/tiff/reader.go @@ -1340,6 +1365,12 @@ go_net_textproto_files = \ go/net/textproto/textproto.go \ go/net/textproto/writer.go +go_old_netchan_files = \ + go/old/netchan/common.go \ + go/old/netchan/export.go \ + go/old/netchan/import.go +go_old_regexp_files = \ + go/old/regexp/regexp.go go_old_template_files = \ go/old/template/doc.go \ go/old/template/execute.go \ @@ -1361,6 +1392,14 @@ go_path_filepath_files = \ go/path/filepath/path.go \ go/path/filepath/path_unix.go +go_regexp_syntax_files = \ + go/regexp/syntax/compile.go \ + go/regexp/syntax/parse.go \ + go/regexp/syntax/perl_groups.go \ + go/regexp/syntax/prog.go \ + go/regexp/syntax/regexp.go \ + go/regexp/syntax/simplify.go + go_rpc_jsonrpc_files = \ go/rpc/jsonrpc/client.go \ go/rpc/jsonrpc/server.go @@ -1559,7 +1598,6 @@ libgo_go_objs = \ cmath/cmath.lo \ crypto/crypto.lo \ csv/csv.lo \ - ebnf/ebnf.lo \ exec/exec.lo \ expvar/expvar.lo \ flag/flag.lo \ @@ -1576,7 +1614,6 @@ libgo_go_objs = \ mail/mail.lo \ mime/mime.lo \ net/net.lo \ - netchan/netchan.lo \ os/os.lo \ patch/patch.lo \ path/path.lo \ @@ -1596,7 +1633,6 @@ libgo_go_objs = \ tabwriter/tabwriter.lo \ template/template.lo \ time/time.lo \ - try/try.lo \ unicode/unicode.lo \ url/url.lo \ utf16/utf16.lo \ @@ -1613,8 +1649,8 @@ libgo_go_objs = \ container/heap.lo \ container/list.lo \ container/ring.lo \ - container/vector.lo \ crypto/aes.lo \ + crypto/bcrypt.lo \ crypto/blowfish.lo \ crypto/cast5.lo \ crypto/cipher.lo \ @@ -1657,11 +1693,16 @@ libgo_go_objs = \ encoding/git85.lo \ encoding/hex.lo \ encoding/pem.lo \ + exp/ebnf.lo \ exp/gui.lo \ exp/norm.lo \ - exp/regexp.lo \ + exp/spdy.lo \ + exp/sql.lo \ + exp/ssh.lo \ + exp/terminal.lo \ + exp/types.lo \ exp/gui/x11.lo \ - exp/regexp/syntax.lo \ + exp/sql/driver.lo \ exp/template/html.lo \ go/ast.lo \ go/build.lo \ @@ -1670,8 +1711,6 @@ libgo_go_objs = \ go/printer.lo \ go/scanner.lo \ go/token.lo \ - go/typechecker.lo \ - go/types.lo \ hash/adler32.lo \ hash/crc32.lo \ hash/crc64.lo \ @@ -1680,8 +1719,8 @@ libgo_go_objs = \ http/fcgi.lo \ http/httptest.lo \ http/pprof.lo \ - http/spdy.lo \ image/bmp.lo \ + image/color.lo \ image/draw.lo \ image/gif.lo \ image/jpeg.lo \ @@ -1693,11 +1732,14 @@ libgo_go_objs = \ mime/multipart.lo \ net/dict.lo \ net/textproto.lo \ + old/netchan.lo \ + old/regexp.lo \ old/template.lo \ $(os_lib_inotify_lo) \ os/user.lo \ os/signal.lo \ path/filepath.lo \ + regexp/syntax.lo \ rpc/jsonrpc.lo \ runtime/debug.lo \ runtime/pprof.lo \ @@ -1853,13 +1895,6 @@ csv/check: $(CHECK_DEPS) @$(CHECK) .PHONY: csv/check -ebnf/ebnf.lo: $(go_ebnf_files) container/vector.gox go/scanner.gox \ - go/token.gox os.gox strconv.gox unicode.gox utf8.gox - $(BUILDPACKAGE) -ebnf/check: $(CHECK_DEPS) - @$(CHECK) -.PHONY: ebnf/check - exec/exec.lo: $(go_exec_files) bytes.gox io.gox os.gox strconv.gox \ strings.gox syscall.gox $(BUILDPACKAGE) @@ -1881,7 +1916,7 @@ flag/check: $(CHECK_DEPS) .PHONY: flag/check fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox math.gox os.gox reflect.gox \ - strconv.gox strings.gox unicode.gox utf8.gox + strconv.gox strings.gox sync.gox unicode.gox utf8.gox $(BUILDPACKAGE) fmt/check: $(CHECK_DEPS) @$(CHECK) @@ -1901,8 +1936,8 @@ hash/check: $(CHECK_DEPS) @$(CHECK) .PHONY: hash/check -html/html.lo: $(go_html_files) bytes.gox io.gox os.gox strconv.gox strings.gox \ - utf8.gox +html/html.lo: $(go_html_files) bufio.gox bytes.gox fmt.gox io.gox os.gox \ + strconv.gox strings.gox utf8.gox $(BUILDPACKAGE) html/check: $(CHECK_DEPS) @$(CHECK) @@ -1919,7 +1954,8 @@ http/check: $(CHECK_DEPS) @$(CHECK) .PHONY: http/check -image/image.lo: $(go_image_files) bufio.gox io.gox os.gox strconv.gox +image/image.lo: $(go_image_files) bufio.gox image/color.gox io.gox os.gox \ + strconv.gox $(BUILDPACKAGE) image/check: $(CHECK_DEPS) @$(CHECK) @@ -1975,13 +2011,6 @@ net/check: $(CHECK_DEPS) @$(CHECK) .PHONY: net/check -netchan/netchan.lo: $(go_netchan_files) gob.gox io.gox log.gox net.gox os.gox \ - reflect.gox strconv.gox sync.gox time.gox - $(BUILDPACKAGE) -netchan/check: $(CHECK_DEPS) - @$(CHECK) -.PHONY: netchan/check - os/os.lo: $(go_os_files) runtime.gox sync.gox syscall.gox $(BUILDPACKAGE) os/check: $(CHECK_DEPS) @@ -2019,8 +2048,8 @@ reflect/check: $(CHECK_DEPS) @$(CHECK) .PHONY: reflect/check -regexp/regexp.lo: $(go_regexp_files) bytes.gox io.gox os.gox strings.gox \ - utf8.gox +regexp/regexp.lo: $(go_regexp_files) bytes.gox io.gox os.gox \ + regexp/syntax.gox strconv.gox strings.gox sync.gox utf8.gox $(BUILDPACKAGE) regexp/check: $(CHECK_DEPS) @$(CHECK) @@ -2067,7 +2096,7 @@ strconv/check: $(CHECK_DEPS) @$(CHECK) .PHONY: strconv/check -strings/strings.lo: $(go_strings_files) os.gox unicode.gox utf8.gox +strings/strings.lo: $(go_strings_files) io.gox os.gox unicode.gox utf8.gox $(BUILDPACKAGE) strings/check: $(CHECK_DEPS) @$(CHECK) @@ -2102,8 +2131,9 @@ template/check: $(CHECK_DEPS) @$(CHECK) .PHONY: template/check -testing/testing.lo: $(go_testing_files) flag.gox fmt.gox os.gox regexp.gox \ - runtime.gox runtime/pprof.gox strings.gox strconv.gox time.gox +testing/testing.lo: $(go_testing_files) bytes.gox flag.gox fmt.gox io.gox \ + os.gox regexp.gox runtime.gox runtime/pprof.gox strings.gox \ + strconv.gox time.gox $(BUILDPACKAGE) testing/check: $(CHECK_DEPS) @$(CHECK) @@ -2116,12 +2146,6 @@ time/check: $(CHECK_DEPS) @$(CHECK) .PHONY: time/check -try/try.lo: $(go_try_files) fmt.gox io.gox os.gox reflect.gox unicode.gox - $(BUILDPACKAGE) -try/check: $(CHECK_DEPS) - @$(CHECK) -.PHONY: try/check - unicode/unicode.lo: $(go_unicode_files) $(BUILDPACKAGE) unicode/check: $(CHECK_DEPS) @@ -2147,8 +2171,10 @@ utf8/check: $(CHECK_DEPS) .PHONY: utf8/check websocket/websocket.lo: $(go_websocket_files) bufio.gox bytes.gox \ - crypto/md5.gox crypto/tls.gox encoding/binary.gox fmt.gox \ - http.gox io.gox net.gox os.gox rand.gox strings.gox url.gox + crypto/md5.gox crypto/rand.gox crypto/sha1.gox crypto/tls.gox \ + encoding/base64.gox encoding/binary.gox fmt.gox http.gox \ + io.gox io/ioutil.gox json.gox net.gox os.gox rand.gox \ + strings.gox strconv.gox sync.gox url.gox $(BUILDPACKAGE) websocket/check: $(CHECK_DEPS) @$(CHECK) @@ -2237,13 +2263,6 @@ container/ring/check: $(CHECK_DEPS) @$(CHECK) .PHONY: container/ring/check -container/vector.lo: $(go_container_vector_files) - $(BUILDPACKAGE) -container/vector/check: $(CHECK_DEPS) - @$(MKDIR_P) container/vector - @$(CHECK) -.PHONY: container/vector/check - crypto/aes.lo: $(go_crypto_aes_files) os.gox strconv.gox $(BUILDPACKAGE) crypto/aes/check: $(CHECK_DEPS) @@ -2251,6 +2270,15 @@ crypto/aes/check: $(CHECK_DEPS) @$(CHECK) .PHONY: crypto/aes/check +crypto/bcrypt.lo: $(go_crypto_bcrypt_files) crypto/blowfish.gox \ + crypto/rand.gox crypto/subtle.gox encoding/base64.gox \ + fmt.gox io.gox os.gox strconv.gox + $(BUILDPACKAGE) +crypto/bcrypt/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/bcrypt + @$(CHECK) +.PHONY: crypto/bcrypt/check + crypto/blowfish.lo: $(go_crypto_blowfish_files) os.gox strconv.gox $(BUILDPACKAGE) crypto/blowfish/check: $(CHECK_DEPS) @@ -2403,12 +2431,12 @@ crypto/subtle/check: $(CHECK_DEPS) .PHONY: crypto/subtle/check crypto/tls.lo: $(go_crypto_tls_files) big.gox bytes.gox crypto.gox \ - crypto/aes.gox crypto/cipher.gox crypto/elliptic.gox \ - crypto/hmac.gox crypto/md5.gox crypto/rand.gox crypto/rc4.gox \ - crypto/rsa.gox crypto/sha1.gox crypto/subtle.gox \ - crypto/x509.gox crypto/x509/pkix.gox encoding/pem.gox \ - hash.gox io.gox io/ioutil.gox net.gox os.gox strconv.gox \ - strings.gox sync.gox time.gox + crypto/aes.gox crypto/cipher.gox crypto/des.gox \ + crypto/elliptic.gox crypto/hmac.gox crypto/md5.gox \ + crypto/rand.gox crypto/rc4.gox crypto/rsa.gox crypto/sha1.gox \ + crypto/subtle.gox crypto/x509.gox crypto/x509/pkix.gox \ + encoding/pem.gox hash.gox io.gox io/ioutil.gox net.gox os.gox \ + strconv.gox strings.gox sync.gox time.gox $(BUILDPACKAGE) crypto/tls/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/tls @@ -2583,6 +2611,14 @@ encoding/pem/check: $(CHECK_DEPS) @$(CHECK) .PHONY: encoding/pem/check +exp/ebnf.lo: $(go_exp_ebnf_files) go/scanner.gox go/token.gox os.gox \ + strconv.gox unicode.gox utf8.gox + $(BUILDPACKAGE) +exp/ebnf/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/ebnf + @$(CHECK) +.PHONY: exp/ebnf/check + exp/gui.lo: $(go_exp_gui_files) image.gox image/draw.gox os.gox $(BUILDPACKAGE) exp/gui/check: $(CHECK_DEPS) @@ -2590,20 +2626,55 @@ exp/gui/check: $(CHECK_DEPS) @$(CHECK) .PHONY: exp/gui/check -exp/norm.lo: $(go_exp_norm_files) utf8.gox +exp/norm.lo: $(go_exp_norm_files) io.gox os.gox utf8.gox $(BUILDPACKAGE) exp/norm/check: $(CHECK_DEPS) @$(MKDIR_P) exp/norm @$(CHECK) .PHONY: exp/norm/check -exp/regexp.lo: $(go_exp_regexp_files) bytes.gox exp/regexp/syntax.gox io.gox \ - os.gox strings.gox sync.gox utf8.gox +exp/spdy.lo: $(go_exp_spdy_files) bytes.gox compress/zlib.gox \ + encoding/binary.gox http.gox io.gox os.gox strings.gox + $(BUILDPACKAGE) +exp/spdy/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/spdy + @$(CHECK) +.PHONY: exp/spdy/check + +exp/sql.lo: $(go_exp_sql_files) exp/sql/driver.gox fmt.gox os.gox reflect.gox \ + runtime.gox strconv.gox sync.gox + $(BUILDPACKAGE) +exp/sql/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/sql + @$(CHECK) +.PHONY: exp/sql/check + +exp/ssh.lo: $(go_exp_ssh_files) big.gox bufio.gox bytes.gox crypto.gox \ + crypto/aes.gox crypto/cipher.gox crypto/hmac.gox \ + crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \ + crypto/subtle.gox crypto/x509.gox encoding/pem.gox hash.gox \ + io.gox net.gox os.gox reflect.gox strconv.gox sync.gox + $(BUILDPACKAGE) +exp/ssh/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/ssh + @$(CHECK) +.PHONY: exp/ssh/check + +exp/terminal.lo: $(go_exp_terminal_files) io.gox os.gox syscall.gox + $(BUILDPACKAGE) +exp/terminal/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/terminal + @$(CHECK) +.PHONY: exp/terminal/check + +exp/types.lo: $(go_exp_types_files) big.gox bufio.gox fmt.gox go/ast.gox \ + go/token.gox io.gox os.gox path/filepath.gox runtime.gox \ + scanner.gox sort.gox strconv.gox strings.gox $(BUILDPACKAGE) -exp/regexp/check: $(CHECK_DEPS) - @$(MKDIR_P) exp/regexp +exp/types/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/types @$(CHECK) -.PHONY: exp/regexp/check +.PHONY: exp/types/check exp/gui/x11.lo: $(go_exp_gui_x11_files) bufio.gox exp/gui.gox image.gox \ image/draw.gox io.gox log.gox net.gox os.gox strconv.gox \ @@ -2614,15 +2685,17 @@ exp/gui/x11/check: $(CHECK_DEPS) @$(CHECK) .PHONY: exp/gui/x11/check -exp/regexp/syntax.lo: $(go_exp_regexp_syntax_files) bytes.gox os.gox sort.gox strconv.gox strings.gox unicode.gox utf8.gox +exp/sql/driver.lo: $(go_exp_sql_driver_files) fmt.gox os.gox reflect.gox \ + strconv.gox $(BUILDPACKAGE) -exp/regexp/syntax/check: $(CHECK_DEPS) - @$(MKDIR_P) exp/regexp/syntax +exp/sql/driver/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/sql/driver @$(CHECK) -.PHONY: exp/regexp/syntax/check +.PHONY: exp/sql/driver/check -exp/template/html.lo: $(go_exp_template_html_files) fmt.gox template.gox \ - template/parse.gox +exp/template/html.lo: $(go_exp_template_html_files) bytes.gox fmt.gox \ + html.gox json.gox os.gox strings.gox template.gox \ + template/parse.gox unicode.gox utf8.gox $(BUILDPACKAGE) exp/template/html/check: $(CHECK_DEPS) @$(MKDIR_P) exp/template/html @@ -2637,9 +2710,10 @@ go/ast/check: $(CHECK_DEPS) @$(CHECK) .PHONY: go/ast/check -go/build.lo: $(go_go_build_files) bytes.gox exec.gox fmt.gox go/parser.gox \ - go/token.gox log.gox os.gox path/filepath.gox regexp.gox \ - runtime.gox strconv.gox strings.gox runtime.gox +go/build.lo: $(go_go_build_files) bytes.gox exec.gox fmt.gox go/ast.gox \ + go/doc.gox go/parser.gox go/token.gox io/ioutil.gox log.gox \ + os.gox path.gox path/filepath.gox regexp.gox runtime.gox \ + sort.gox strconv.gox strings.gox runtime.gox unicode.gox $(BUILDPACKAGE) go/build/check: $(CHECK_DEPS) @$(MKDIR_P) go/build @@ -2656,7 +2730,7 @@ s-syslist: Makefile $(STAMP) $@ go/doc.lo: $(go_go_doc_files) go/ast.gox go/token.gox io.gox regexp.gox \ - sort.gox strings.gox template.gox + sort.gox strings.gox template.gox unicode.gox utf8.gox $(BUILDPACKAGE) go/doc/check: $(CHECK_DEPS) @$(MKDIR_P) go/doc @@ -2674,7 +2748,7 @@ go/parser/check: $(CHECK_DEPS) go/printer.lo: $(go_go_printer_files) bytes.gox fmt.gox go/ast.gox \ go/token.gox io.gox os.gox path/filepath.gox reflect.gox \ - runtime.gox strings.gox tabwriter.gox + strings.gox tabwriter.gox $(BUILDPACKAGE) go/printer/check: $(CHECK_DEPS) @$(MKDIR_P) go/printer @@ -2690,30 +2764,14 @@ go/scanner/check: $(CHECK_DEPS) @$(CHECK) .PHONY: go/scanner/check -go/token.lo: $(go_go_token_files) fmt.gox strconv.gox +go/token.lo: $(go_go_token_files) fmt.gox gob.gox io.gox os.gox sort.gox \ + strconv.gox sync.gox $(BUILDPACKAGE) go/token/check: $(CHECK_DEPS) @$(MKDIR_P) go/token @$(CHECK) .PHONY: go/token/check -go/typechecker.lo: $(go_go_typechecker_files) fmt.gox go/ast.gox go/token.gox \ - go/scanner.gox os.gox - $(BUILDPACKAGE) -go/typechecker/check: $(CHECK_DEPS) - @$(MKDIR_P) go/typechecker - @$(CHECK) -.PHONY: go/typechecker/check - -go/types.lo: $(go_go_types_files) big.gox bufio.gox fmt.gox go/ast.gox \ - go/token.gox io.gox os.gox path/filepath.gox runtime.gox \ - scanner.gox sort.gox strconv.gox strings.gox - $(BUILDPACKAGE) -go/types/check: $(CHECK_DEPS) - @$(MKDIR_P) go/types - @$(CHECK) -.PHONY: go/types/check - hash/adler32.lo: $(go_hash_adler32_files) hash.gox os.gox $(BUILDPACKAGE) hash/adler32/check: $(CHECK_DEPS) @@ -2777,23 +2835,21 @@ http/pprof/check: $(CHECK_DEPS) @$(CHECK) .PHONY: http/pprof/check -http/spdy.lo: $(go_http_spdy_files) bytes.gox compress/zlib.gox \ - encoding/binary.gox http.gox io.gox os.gox strconv.gox \ - strings.gox sync.gox - $(BUILDPACKAGE) -http/spdy/check: $(CHECK_DEPS) - @$(MKDIR_P) http/spdy - @$(CHECK) -.PHONY: http/spdy/check - -image/bmp.lo: $(go_image_bmp_files) image.gox io.gox os.gox +image/bmp.lo: $(go_image_bmp_files) image.gox image/color.gox io.gox os.gox $(BUILDPACKAGE) image/bmp/check: $(CHECK_DEPS) @$(MKDIR_P) image/bmp @$(CHECK) .PHONY: image/bmp/check -image/draw.lo: $(go_image_draw_files) image.gox image/ycbcr.gox +image/color.lo: $(go_image_color_files) + $(BUILDPACKAGE) +image/color/check: $(CHECK_DEPS) + @$(MKDIR_P) image/color + @$(CHECK) +.PHONY: image/color/check + +image/draw.lo: $(go_image_draw_files) image.gox image/color.gox image/ycbcr.gox $(BUILDPACKAGE) image/draw/check: $(CHECK_DEPS) @$(MKDIR_P) image/draw @@ -2801,46 +2857,48 @@ image/draw/check: $(CHECK_DEPS) .PHONY: image/draw/check image/gif.lo: $(go_image_gif_files) bufio.gox compress/lzw.gox fmt.gox \ - image.gox io.gox os.gox + image.gox image/color.gox io.gox os.gox $(BUILDPACKAGE) image/gif/check: $(CHECK_DEPS) @$(MKDIR_P) image/gif @$(CHECK) .PHONY: image/gif/check -image/jpeg.lo: $(go_image_jpeg_files) bufio.gox image.gox image/ycbcr.gox \ - io.gox os.gox +image/jpeg.lo: $(go_image_jpeg_files) bufio.gox image.gox image/color.gox \ + image/ycbcr.gox io.gox os.gox $(BUILDPACKAGE) image/jpeg/check: $(CHECK_DEPS) @$(MKDIR_P) image/jpeg @$(CHECK) .PHONY: image/jpeg/check -image/png.lo: $(go_image_png_files) bufio.gox compress/zlib.gox fmt.gox \ - hash.gox hash/crc32.gox image.gox io.gox os.gox strconv.gox +image/png.lo: $(go_image_png_files) bufio.gox compress/zlib.gox \ + encoding/binary.gox fmt.gox hash.gox hash/crc32.gox image.gox \ + image/color.gox io.gox os.gox strconv.gox $(BUILDPACKAGE) image/png/check: $(CHECK_DEPS) @$(MKDIR_P) image/png @$(CHECK) .PHONY: image/png/check -image/tiff.lo: $(go_image_tiff_files) compress/lzw.gox compress/zlib.gox \ - encoding/binary.gox image.gox io.gox io/ioutil.gox os.gox +image/tiff.lo: $(go_image_tiff_files) bufio.gox compress/lzw.gox \ + compress/zlib.gox encoding/binary.gox image.gox \ + image/color.gox io.gox io/ioutil.gox os.gox $(BUILDPACKAGE) image/tiff/check: $(CHECK_DEPS) @$(MKDIR_P) image/tiff @$(CHECK) .PHONY: image/tiff/check -image/ycbcr.lo: $(go_image_ycbcr_files) image.gox +image/ycbcr.lo: $(go_image_ycbcr_files) image.gox image/color.gox $(BUILDPACKAGE) image/ycbcr/check: $(CHECK_DEPS) @$(MKDIR_P) image/ycbcr @$(CHECK) .PHONY: image/ycbcr/check -index/suffixarray.lo: $(go_index_suffixarray_files) bytes.gox regexp.gox \ - sort.gox +index/suffixarray.lo: $(go_index_suffixarray_files) bytes.gox \ + encoding/binary.gox io.gox os.gox regexp.gox sort.gox $(BUILDPACKAGE) index/suffixarray/check: $(CHECK_DEPS) @$(MKDIR_P) index/suffixarray @@ -2869,13 +2927,30 @@ net/dict.lo: $(go_net_dict_files) net/textproto.gox os.gox strconv.gox \ $(BUILDPACKAGE) net/textproto.lo: $(go_net_textproto_files) bufio.gox bytes.gox fmt.gox \ - io.gox io/ioutil.gox net.gox os.gox strconv.gox sync.gox + io.gox io/ioutil.gox net.gox os.gox strings.gox strconv.gox \ + sync.gox $(BUILDPACKAGE) net/textproto/check: $(CHECK_DEPS) @$(MKDIR_P) net/textproto @$(CHECK) .PHONY: net/textproto/check +old/netchan.lo: $(go_old_netchan_files) gob.gox io.gox log.gox net.gox os.gox \ + reflect.gox strconv.gox sync.gox time.gox + $(BUILDPACKAGE) +old/netchan/check: $(CHECK_DEPS) + @$(MKDIR_P) old/netchan + @$(CHECK) +.PHONY: old/netchan/check + +old/regexp.lo: $(go_old_regexp_files) bytes.gox io.gox os.gox strings.gox \ + utf8.gox + $(BUILDPACKAGE) +old/regexp/check: $(CHECK_DEPS) + @$(MKDIR_P) old/regexp + @$(CHECK) +.PHONY: old/regexp/check + old/template.lo: $(go_old_template_files) bytes.gox fmt.gox io.gox \ io/ioutil.gox os.gox reflect.gox strconv.gox strings.gox \ unicode.gox utf8.gox @@ -2915,6 +2990,14 @@ path/filepath/check: $(CHECK_DEPS) @$(CHECK) .PHONY: path/filepath/check +regexp/syntax.lo: $(go_regexp_syntax_files) bytes.gox os.gox sort.gox \ + strconv.gox strings.gox unicode.gox utf8.gox + $(BUILDPACKAGE) +regexp/syntax/check: $(CHECK_DEPS) + @$(MKDIR_P) regexp/syntax + @$(CHECK) +.PHONY: regexp/syntax/check + rpc/jsonrpc.lo: $(go_rpc_jsonrpc_files) fmt.gox io.gox json.gox net.gox \ os.gox rpc.gox sync.gox $(BUILDPACKAGE) @@ -3005,8 +3088,6 @@ crypto.gox: crypto/crypto.lo $(BUILDGOX) csv.gox: csv/csv.lo $(BUILDGOX) -ebnf.gox: ebnf/ebnf.lo - $(BUILDGOX) exec.gox: exec/exec.lo $(BUILDGOX) expvar.gox: expvar/expvar.lo @@ -3039,8 +3120,6 @@ mime.gox: mime/mime.lo $(BUILDGOX) net.gox: net/net.lo $(BUILDGOX) -netchan.gox: netchan/netchan.lo - $(BUILDGOX) os.gox: os/os.lo $(BUILDGOX) patch.gox: patch/patch.lo @@ -3081,8 +3160,6 @@ testing.gox: testing/testing.lo $(BUILDGOX) time.gox: time/time.lo $(BUILDGOX) -try.gox: try/try.lo - $(BUILDGOX) unicode.gox: unicode/unicode.lo $(BUILDGOX) url.gox: url/url.lo @@ -3118,11 +3195,11 @@ container/list.gox: container/list.lo $(BUILDGOX) container/ring.gox: container/ring.lo $(BUILDGOX) -container/vector.gox: container/vector.lo - $(BUILDGOX) crypto/aes.gox: crypto/aes.lo $(BUILDGOX) +crypto/bcrypt.gox: crypto/bcrypt.lo + $(BUILDGOX) crypto/blowfish.gox: crypto/blowfish.lo $(BUILDGOX) crypto/cast5.gox: crypto/cast5.lo @@ -3212,17 +3289,27 @@ encoding/hex.gox: encoding/hex.lo encoding/pem.gox: encoding/pem.lo $(BUILDGOX) +exp/ebnf.gox: exp/ebnf.lo + $(BUILDGOX) exp/gui.gox: exp/gui.lo $(BUILDGOX) exp/norm.gox: exp/norm.lo $(BUILDGOX) -exp/regexp.gox: exp/regexp.lo +exp/spdy.gox: exp/spdy.lo + $(BUILDGOX) +exp/sql.gox: exp/sql.lo + $(BUILDGOX) +exp/ssh.gox: exp/ssh.lo + $(BUILDGOX) +exp/terminal.gox: exp/terminal.lo + $(BUILDGOX) +exp/types.gox: exp/types.lo $(BUILDGOX) exp/gui/x11.gox: exp/gui/x11.lo $(BUILDGOX) -exp/regexp/syntax.gox: exp/regexp/syntax.lo +exp/sql/driver.gox: exp/sql/driver.lo $(BUILDGOX) exp/template/html.gox: exp/template/html.lo @@ -3242,10 +3329,6 @@ go/scanner.gox: go/scanner.lo $(BUILDGOX) go/token.gox: go/token.lo $(BUILDGOX) -go/typechecker.gox: go/typechecker.lo - $(BUILDGOX) -go/types.gox: go/types.lo - $(BUILDGOX) hash/adler32.gox: hash/adler32.lo $(BUILDGOX) @@ -3264,11 +3347,11 @@ http/httptest.gox: http/httptest.lo $(BUILDGOX) http/pprof.gox: http/pprof.lo $(BUILDGOX) -http/spdy.gox: http/spdy.lo - $(BUILDGOX) image/bmp.gox: image/bmp.lo $(BUILDGOX) +image/color.gox: image/color.lo + $(BUILDGOX) image/draw.gox: image/draw.lo $(BUILDGOX) image/gif.gox: image/gif.lo @@ -3296,6 +3379,10 @@ net/dict.gox: net/dict.lo net/textproto.gox: net/textproto.lo $(BUILDGOX) +old/netchan.gox: old/netchan.lo + $(BUILDGOX) +old/regexp.gox: old/regexp.lo + $(BUILDGOX) old/template.gox: old/template.lo $(BUILDGOX) @@ -3309,6 +3396,9 @@ os/signal.gox: os/signal.lo path/filepath.gox: path/filepath.lo $(BUILDGOX) +regexp/syntax.gox: regexp/syntax.lo + $(BUILDGOX) + rpc/jsonrpc.gox: rpc/jsonrpc.lo $(BUILDGOX) @@ -3344,7 +3434,6 @@ TEST_PACKAGES = \ bytes/check \ cmath/check \ csv/check \ - ebnf/check \ exec/check \ expvar/check \ flag/check \ @@ -3360,7 +3449,6 @@ TEST_PACKAGES = \ mail/check \ mime/check \ net/check \ - netchan/check \ os/check \ patch/check \ path/check \ @@ -3379,7 +3467,6 @@ TEST_PACKAGES = \ tabwriter/check \ template/check \ time/check \ - try/check \ unicode/check \ url/check \ utf16/check \ @@ -3396,8 +3483,8 @@ TEST_PACKAGES = \ container/heap/check \ container/list/check \ container/ring/check \ - container/vector/check \ crypto/aes/check \ + crypto/bcrypt/check \ crypto/blowfish/check \ crypto/cast5/check \ crypto/cipher/check \ @@ -3437,9 +3524,12 @@ TEST_PACKAGES = \ encoding/git85/check \ encoding/hex/check \ encoding/pem/check \ + exp/ebnf/check \ exp/norm/check \ - exp/regexp/check \ - exp/regexp/syntax/check \ + exp/spdy/check \ + exp/sql/check \ + exp/ssh/check \ + exp/terminal/check \ exp/template/html/check \ go/ast/check \ $(go_build_check_omitted_since_it_calls_6g) \ @@ -3447,7 +3537,6 @@ TEST_PACKAGES = \ go/printer/check \ go/scanner/check \ go/token/check \ - go/typechecker/check \ $(go_types_check_omitted_since_it_calls_6g) \ hash/adler32/check \ hash/crc32/check \ @@ -3455,7 +3544,6 @@ TEST_PACKAGES = \ hash/fnv/check \ http/cgi/check \ http/fcgi/check \ - http/spdy/check \ image/draw/check \ image/jpeg/check \ image/png/check \ @@ -3465,11 +3553,14 @@ TEST_PACKAGES = \ io/ioutil/check \ mime/multipart/check \ net/textproto/check \ + old/netchan/check \ + old/regexp/check \ old/template/check \ $(os_inotify_check) \ os/user/check \ os/signal/check \ path/filepath/check \ + regexp/syntax/check \ rpc/jsonrpc/check \ sync/atomic/check \ template/parse/check \ diff --git a/libgo/Makefile.in b/libgo/Makefile.in index e18d242..05223a6 100644 --- a/libgo/Makefile.in +++ b/libgo/Makefile.in @@ -103,7 +103,7 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \ "$(DESTDIR)$(toolexeclibgoencodingdir)" \ "$(DESTDIR)$(toolexeclibgoexpdir)" \ "$(DESTDIR)$(toolexeclibgoexpguidir)" \ - "$(DESTDIR)$(toolexeclibgoexpregexpdir)" \ + "$(DESTDIR)$(toolexeclibgoexpsqldir)" \ "$(DESTDIR)$(toolexeclibgoexptemplatedir)" \ "$(DESTDIR)$(toolexeclibgogodir)" \ "$(DESTDIR)$(toolexeclibgohashdir)" \ @@ -116,6 +116,7 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \ "$(DESTDIR)$(toolexeclibgoolddir)" \ "$(DESTDIR)$(toolexeclibgoosdir)" \ "$(DESTDIR)$(toolexeclibgopathdir)" \ + "$(DESTDIR)$(toolexeclibgoregexpdir)" \ "$(DESTDIR)$(toolexeclibgorpcdir)" \ "$(DESTDIR)$(toolexeclibgoruntimedir)" \ "$(DESTDIR)$(toolexeclibgosyncdir)" \ @@ -131,22 +132,21 @@ LTLIBRARIES = $(toolexeclib_LTLIBRARIES) am__DEPENDENCIES_1 = am__DEPENDENCIES_2 = asn1/asn1.lo big/big.lo bufio/bufio.lo \ bytes/bytes.lo bytes/index.lo cmath/cmath.lo crypto/crypto.lo \ - csv/csv.lo ebnf/ebnf.lo exec/exec.lo expvar/expvar.lo \ - flag/flag.lo fmt/fmt.lo gob/gob.lo hash/hash.lo html/html.lo \ - http/http.lo image/image.lo io/io.lo json/json.lo log/log.lo \ - math/math.lo mail/mail.lo mime/mime.lo net/net.lo \ - netchan/netchan.lo os/os.lo patch/patch.lo path/path.lo \ - rand/rand.lo reflect/reflect.lo regexp/regexp.lo rpc/rpc.lo \ - runtime/runtime.lo scanner/scanner.lo smtp/smtp.lo \ + csv/csv.lo exec/exec.lo expvar/expvar.lo flag/flag.lo \ + fmt/fmt.lo gob/gob.lo hash/hash.lo html/html.lo http/http.lo \ + image/image.lo io/io.lo json/json.lo log/log.lo math/math.lo \ + mail/mail.lo mime/mime.lo net/net.lo os/os.lo patch/patch.lo \ + path/path.lo rand/rand.lo reflect/reflect.lo regexp/regexp.lo \ + rpc/rpc.lo runtime/runtime.lo scanner/scanner.lo smtp/smtp.lo \ sort/sort.lo strconv/strconv.lo strings/strings.lo \ sync/sync.lo syslog/syslog.lo syslog/syslog_c.lo \ tabwriter/tabwriter.lo template/template.lo time/time.lo \ - try/try.lo unicode/unicode.lo url/url.lo utf16/utf16.lo \ - utf8/utf8.lo websocket/websocket.lo xml/xml.lo archive/tar.lo \ + unicode/unicode.lo url/url.lo utf16/utf16.lo utf8/utf8.lo \ + websocket/websocket.lo xml/xml.lo archive/tar.lo \ archive/zip.lo compress/bzip2.lo compress/flate.lo \ compress/gzip.lo compress/lzw.lo compress/zlib.lo \ container/heap.lo container/list.lo container/ring.lo \ - container/vector.lo crypto/aes.lo crypto/blowfish.lo \ + crypto/aes.lo crypto/bcrypt.lo crypto/blowfish.lo \ crypto/cast5.lo crypto/cipher.lo crypto/des.lo crypto/dsa.lo \ crypto/ecdsa.lo crypto/elliptic.lo crypto/hmac.lo \ crypto/md4.lo crypto/md5.lo crypto/ocsp.lo crypto/openpgp.lo \ @@ -159,22 +159,23 @@ am__DEPENDENCIES_2 = asn1/asn1.lo big/big.lo bufio/bufio.lo \ crypto/x509/pkix.lo debug/dwarf.lo debug/elf.lo debug/gosym.lo \ debug/macho.lo debug/pe.lo encoding/ascii85.lo \ encoding/base32.lo encoding/base64.lo encoding/binary.lo \ - encoding/git85.lo encoding/hex.lo encoding/pem.lo exp/gui.lo \ - exp/norm.lo exp/regexp.lo exp/gui/x11.lo exp/regexp/syntax.lo \ + encoding/git85.lo encoding/hex.lo encoding/pem.lo exp/ebnf.lo \ + exp/gui.lo exp/norm.lo exp/spdy.lo exp/sql.lo exp/ssh.lo \ + exp/terminal.lo exp/types.lo exp/gui/x11.lo exp/sql/driver.lo \ exp/template/html.lo go/ast.lo go/build.lo go/doc.lo \ go/parser.lo go/printer.lo go/scanner.lo go/token.lo \ - go/typechecker.lo go/types.lo hash/adler32.lo hash/crc32.lo \ - hash/crc64.lo hash/fnv.lo http/cgi.lo http/fcgi.lo \ - http/httptest.lo http/pprof.lo http/spdy.lo image/bmp.lo \ - image/draw.lo image/gif.lo image/jpeg.lo image/png.lo \ - image/tiff.lo image/ycbcr.lo index/suffixarray.lo io/ioutil.lo \ - mime/multipart.lo net/dict.lo net/textproto.lo old/template.lo \ - $(am__DEPENDENCIES_1) os/user.lo os/signal.lo path/filepath.lo \ - rpc/jsonrpc.lo runtime/debug.lo runtime/pprof.lo \ - sync/atomic.lo sync/atomic_c.lo syscall/syscall.lo \ - syscall/errno.lo syscall/wait.lo template/parse.lo \ - testing/testing.lo testing/iotest.lo testing/quick.lo \ - testing/script.lo + hash/adler32.lo hash/crc32.lo hash/crc64.lo hash/fnv.lo \ + http/cgi.lo http/fcgi.lo http/httptest.lo http/pprof.lo \ + image/bmp.lo image/color.lo image/draw.lo image/gif.lo \ + image/jpeg.lo image/png.lo image/tiff.lo image/ycbcr.lo \ + index/suffixarray.lo io/ioutil.lo mime/multipart.lo \ + net/dict.lo net/textproto.lo old/netchan.lo old/regexp.lo \ + old/template.lo $(am__DEPENDENCIES_1) os/user.lo os/signal.lo \ + path/filepath.lo regexp/syntax.lo rpc/jsonrpc.lo \ + runtime/debug.lo runtime/pprof.lo sync/atomic.lo \ + sync/atomic_c.lo syscall/syscall.lo syscall/errno.lo \ + syscall/wait.lo template/parse.lo testing/testing.lo \ + testing/iotest.lo testing/quick.lo testing/script.lo libgo_la_DEPENDENCIES = $(am__DEPENDENCIES_2) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) @@ -221,8 +222,8 @@ am__libgo_la_SOURCES_DIST = runtime/go-append.c runtime/go-assert.c \ runtime/mem_posix_memalign.c runtime/mem.c runtime/mfinal.c \ runtime/mfixalloc.c runtime/mgc0.c runtime/mheap.c \ runtime/msize.c runtime/proc.c runtime/thread.c \ - runtime/rtems-task-variable-add.c chan.c iface.c malloc.c \ - map.c mprof.c reflect.c sigqueue.c string.c + runtime/yield.c runtime/rtems-task-variable-add.c chan.c \ + iface.c malloc.c map.c mprof.c reflect.c sigqueue.c string.c @HAVE_SYS_MMAN_H_FALSE@am__objects_1 = mem_posix_memalign.lo @HAVE_SYS_MMAN_H_TRUE@am__objects_1 = mem.lo @LIBGO_IS_RTEMS_TRUE@am__objects_2 = rtems-task-variable-add.lo @@ -253,9 +254,9 @@ am__objects_3 = go-append.lo go-assert.lo go-assert-interface.lo \ go-unreflect.lo go-unsafe-new.lo go-unsafe-newarray.lo \ go-unsafe-pointer.lo go-unwind.lo cpuprof.lo mcache.lo \ mcentral.lo $(am__objects_1) mfinal.lo mfixalloc.lo mgc0.lo \ - mheap.lo msize.lo proc.lo thread.lo $(am__objects_2) chan.lo \ - iface.lo malloc.lo map.lo mprof.lo reflect.lo sigqueue.lo \ - string.lo + mheap.lo msize.lo proc.lo thread.lo yield.lo $(am__objects_2) \ + chan.lo iface.lo malloc.lo map.lo mprof.lo reflect.lo \ + sigqueue.lo string.lo am_libgo_la_OBJECTS = $(am__objects_3) libgo_la_OBJECTS = $(am_libgo_la_OBJECTS) libgo_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ @@ -294,16 +295,16 @@ DATA = $(toolexeclibgo_DATA) $(toolexeclibgoarchive_DATA) \ $(toolexeclibgocrypto_DATA) $(toolexeclibgocryptoopenpgp_DATA) \ $(toolexeclibgocryptox509_DATA) $(toolexeclibgodebug_DATA) \ $(toolexeclibgoencoding_DATA) $(toolexeclibgoexp_DATA) \ - $(toolexeclibgoexpgui_DATA) $(toolexeclibgoexpregexp_DATA) \ + $(toolexeclibgoexpgui_DATA) $(toolexeclibgoexpsql_DATA) \ $(toolexeclibgoexptemplate_DATA) $(toolexeclibgogo_DATA) \ $(toolexeclibgohash_DATA) $(toolexeclibgohttp_DATA) \ $(toolexeclibgoimage_DATA) $(toolexeclibgoindex_DATA) \ $(toolexeclibgoio_DATA) $(toolexeclibgomime_DATA) \ $(toolexeclibgonet_DATA) $(toolexeclibgoold_DATA) \ $(toolexeclibgoos_DATA) $(toolexeclibgopath_DATA) \ - $(toolexeclibgorpc_DATA) $(toolexeclibgoruntime_DATA) \ - $(toolexeclibgosync_DATA) $(toolexeclibgotemplate_DATA) \ - $(toolexeclibgotesting_DATA) + $(toolexeclibgoregexp_DATA) $(toolexeclibgorpc_DATA) \ + $(toolexeclibgoruntime_DATA) $(toolexeclibgosync_DATA) \ + $(toolexeclibgotemplate_DATA) $(toolexeclibgotesting_DATA) RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ distclean-recursive maintainer-clean-recursive AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ @@ -575,7 +576,6 @@ toolexeclibgo_DATA = \ cmath.gox \ crypto.gox \ csv.gox \ - ebnf.gox \ exec.gox \ expvar.gox \ flag.gox \ @@ -592,7 +592,6 @@ toolexeclibgo_DATA = \ mail.gox \ mime.gox \ net.gox \ - netchan.gox \ os.gox \ patch.gox \ path.gox \ @@ -613,7 +612,6 @@ toolexeclibgo_DATA = \ template.gox \ testing.gox \ time.gox \ - try.gox \ unicode.gox \ url.gox \ utf16.gox \ @@ -638,12 +636,12 @@ toolexeclibgocontainerdir = $(toolexeclibgodir)/container toolexeclibgocontainer_DATA = \ container/heap.gox \ container/list.gox \ - container/ring.gox \ - container/vector.gox + container/ring.gox toolexeclibgocryptodir = $(toolexeclibgodir)/crypto toolexeclibgocrypto_DATA = \ crypto/aes.gox \ + crypto/bcrypt.gox \ crypto/blowfish.gox \ crypto/cast5.gox \ crypto/cipher.gox \ @@ -701,17 +699,22 @@ toolexeclibgoencoding_DATA = \ toolexeclibgoexpdir = $(toolexeclibgodir)/exp toolexeclibgoexp_DATA = \ + exp/ebnf.gox \ exp/gui.gox \ exp/norm.gox \ - exp/regexp.gox + exp/spdy.gox \ + exp/sql.gox \ + exp/ssh.gox \ + exp/terminal.gox \ + exp/types.gox toolexeclibgoexpguidir = $(toolexeclibgoexpdir)/gui toolexeclibgoexpgui_DATA = \ exp/gui/x11.gox -toolexeclibgoexpregexpdir = $(toolexeclibgoexpdir)/regexp -toolexeclibgoexpregexp_DATA = \ - exp/regexp/syntax.gox +toolexeclibgoexpsqldir = $(toolexeclibgoexpdir)/sql +toolexeclibgoexpsql_DATA = \ + exp/sql/driver.gox toolexeclibgoexptemplatedir = $(toolexeclibgoexpdir)/template toolexeclibgoexptemplate_DATA = \ @@ -725,9 +728,7 @@ toolexeclibgogo_DATA = \ go/parser.gox \ go/printer.gox \ go/scanner.gox \ - go/token.gox \ - go/typechecker.gox \ - go/types.gox + go/token.gox toolexeclibgohashdir = $(toolexeclibgodir)/hash toolexeclibgohash_DATA = \ @@ -741,12 +742,12 @@ toolexeclibgohttp_DATA = \ http/cgi.gox \ http/fcgi.gox \ http/httptest.gox \ - http/pprof.gox \ - http/spdy.gox + http/pprof.gox toolexeclibgoimagedir = $(toolexeclibgodir)/image toolexeclibgoimage_DATA = \ image/bmp.gox \ + image/color.gox \ image/draw.gox \ image/gif.gox \ image/jpeg.gox \ @@ -773,6 +774,8 @@ toolexeclibgonet_DATA = \ toolexeclibgoolddir = $(toolexeclibgodir)/old toolexeclibgoold_DATA = \ + old/netchan.gox \ + old/regexp.gox \ old/template.gox toolexeclibgoosdir = $(toolexeclibgodir)/os @@ -789,6 +792,10 @@ toolexeclibgopathdir = $(toolexeclibgodir)/path toolexeclibgopath_DATA = \ path/filepath.gox +toolexeclibgoregexpdir = $(toolexeclibgodir)/regexp +toolexeclibgoregexp_DATA = \ + regexp/syntax.gox + toolexeclibgorpcdir = $(toolexeclibgodir)/rpc toolexeclibgorpc_DATA = \ rpc/jsonrpc.gox @@ -908,6 +915,7 @@ runtime_files = \ runtime/msize.c \ runtime/proc.c \ runtime/thread.c \ + runtime/yield.c \ $(rtems_task_variable_add_file) \ chan.c \ iface.c \ @@ -963,10 +971,6 @@ go_csv_files = \ go/csv/reader.go \ go/csv/writer.go -go_ebnf_files = \ - go/ebnf/ebnf.go \ - go/ebnf/parser.go - go_exec_files = \ go/exec/exec.go \ go/exec/lp_unix.go @@ -1002,6 +1006,7 @@ go_html_files = \ go/html/escape.go \ go/html/node.go \ go/html/parse.go \ + go/html/render.go \ go/html/token.go go_http_files = \ @@ -1009,6 +1014,7 @@ go_http_files = \ go/http/client.go \ go/http/cookie.go \ go/http/dump.go \ + go/http/filetransport.go \ go/http/fs.go \ go/http/header.go \ go/http/lex.go \ @@ -1023,7 +1029,6 @@ go_http_files = \ go/http/transport.go go_image_files = \ - go/image/color.go \ go/image/format.go \ go/image/geom.go \ go/image/image.go \ @@ -1046,6 +1051,7 @@ go_log_files = \ go/log/log.go go_math_files = \ + go/math/abs.go \ go/math/acosh.go \ go/math/asin.go \ go/math/asinh.go \ @@ -1056,15 +1062,13 @@ go_math_files = \ go/math/cbrt.go \ go/math/const.go \ go/math/copysign.go \ + go/math/dim.go \ go/math/erf.go \ go/math/exp.go \ go/math/exp_port.go \ go/math/exp2.go \ go/math/expm1.go \ - go/math/fabs.go \ - go/math/fdim.go \ go/math/floor.go \ - go/math/fmod.go \ go/math/frexp.go \ go/math/gamma.go \ go/math/hypot.go \ @@ -1078,6 +1082,7 @@ go_math_files = \ go/math/log1p.go \ go/math/log10.go \ go/math/logb.go \ + go/math/mod.go \ go/math/modf.go \ go/math/nextafter.go \ go/math/pow.go \ @@ -1156,11 +1161,6 @@ go_net_files = \ go/net/unixsock.go \ go/net/unixsock_posix.go -go_netchan_files = \ - go/netchan/common.go \ - go/netchan/export.go \ - go/netchan/import.go - @LIBGO_IS_386_FALSE@@LIBGO_IS_SOLARIS_TRUE@@LIBGO_IS_SPARC_FALSE@go_os_dir_file = go/os/dir_regfile.go @LIBGO_IS_386_FALSE@@LIBGO_IS_SOLARIS_TRUE@@LIBGO_IS_SPARC_TRUE@go_os_dir_file = go/os/dir_largefile.go @LIBGO_IS_386_TRUE@@LIBGO_IS_SOLARIS_TRUE@go_os_dir_file = go/os/dir_largefile.go @@ -1218,6 +1218,7 @@ go_reflect_files = \ go/reflect/value.go go_regexp_files = \ + go/regexp/exec.go \ go/regexp/regexp.go go_rpc_files = \ @@ -1257,6 +1258,7 @@ go_strconv_files = \ go_strings_files = \ go/strings/reader.go \ + go/strings/replace.go \ go/strings/strings.go go_sync_files = \ @@ -1289,21 +1291,19 @@ go_template_files = \ go_testing_files = \ go/testing/benchmark.go \ + go/testing/example.go \ go/testing/testing.go go_time_files = \ go/time/format.go \ go/time/sleep.go \ go/time/sys.go \ - go/time/sys_posix.go \ + go/time/sys_unix.go \ go/time/tick.go \ go/time/time.go \ go/time/zoneinfo_posix.go \ go/time/zoneinfo_unix.go -go_try_files = \ - go/try/try.go - go_unicode_files = \ go/unicode/casetables.go \ go/unicode/digit.go \ @@ -1323,6 +1323,8 @@ go_utf8_files = \ go_websocket_files = \ go/websocket/client.go \ + go/websocket/hixie.go \ + go/websocket/hybi.go \ go/websocket/server.go \ go/websocket/websocket.go @@ -1377,17 +1379,15 @@ go_container_list_files = \ go_container_ring_files = \ go/container/ring/ring.go -go_container_vector_files = \ - go/container/vector/defs.go \ - go/container/vector/intvector.go \ - go/container/vector/stringvector.go \ - go/container/vector/vector.go - go_crypto_aes_files = \ go/crypto/aes/block.go \ go/crypto/aes/cipher.go \ go/crypto/aes/const.go +go_crypto_bcrypt_files = \ + go/crypto/bcrypt/base64.go \ + go/crypto/bcrypt/bcrypt.go + go_crypto_blowfish_files = \ go/crypto/blowfish/block.go \ go/crypto/blowfish/const.go \ @@ -1480,6 +1480,7 @@ go_crypto_tls_files = \ go/crypto/tls/handshake_server.go \ go/crypto/tls/key_agreement.go \ go/crypto/tls/prf.go \ + go/crypto/tls/root_unix.go \ go/crypto/tls/tls.go go_crypto_twofish_files = \ @@ -1487,6 +1488,7 @@ go_crypto_twofish_files = \ go_crypto_x509_files = \ go/crypto/x509/cert_pool.go \ + go/crypto/x509/pkcs1.go \ go/crypto/x509/verify.go \ go/crypto/x509/x509.go @@ -1558,7 +1560,8 @@ go_encoding_base64_files = \ go/encoding/base64/base64.go go_encoding_binary_files = \ - go/encoding/binary/binary.go + go/encoding/binary/binary.go \ + go/encoding/binary/varint.go go_encoding_git85_files = \ go/encoding/git85/git.go @@ -1569,35 +1572,73 @@ go_encoding_hex_files = \ go_encoding_pem_files = \ go/encoding/pem/pem.go +go_exp_ebnf_files = \ + go/exp/ebnf/ebnf.go \ + go/exp/ebnf/parser.go + go_exp_gui_files = \ go/exp/gui/gui.go go_exp_norm_files = \ go/exp/norm/composition.go \ go/exp/norm/forminfo.go \ + go/exp/norm/input.go \ go/exp/norm/normalize.go \ + go/exp/norm/readwriter.go \ go/exp/norm/tables.go \ go/exp/norm/trie.go -go_exp_regexp_files = \ - go/exp/regexp/exec.go \ - go/exp/regexp/regexp.go +go_exp_spdy_files = \ + go/exp/spdy/read.go \ + go/exp/spdy/types.go \ + go/exp/spdy/write.go + +go_exp_sql_files = \ + go/exp/sql/convert.go \ + go/exp/sql/sql.go + +go_exp_ssh_files = \ + go/exp/ssh/channel.go \ + go/exp/ssh/common.go \ + go/exp/ssh/doc.go \ + go/exp/ssh/messages.go \ + go/exp/ssh/server.go \ + go/exp/ssh/server_shell.go \ + go/exp/ssh/transport.go + +go_exp_terminal_files = \ + go/exp/terminal/shell.go \ + go/exp/terminal/terminal.go + +go_exp_types_files = \ + go/exp/types/check.go \ + go/exp/types/const.go \ + go/exp/types/exportdata.go \ + go/exp/types/gcimporter.go \ + go/exp/types/types.go \ + go/exp/types/universe.go go_exp_gui_x11_files = \ go/exp/gui/x11/auth.go \ go/exp/gui/x11/conn.go +go_exp_sql_driver_files = \ + go/exp/sql/driver/driver.go \ + go/exp/sql/driver/types.go + go_exp_template_html_files = \ + go/exp/template/html/attr.go \ + go/exp/template/html/clone.go \ + go/exp/template/html/content.go \ go/exp/template/html/context.go \ - go/exp/template/html/escape.go - -go_exp_regexp_syntax_files = \ - go/exp/regexp/syntax/compile.go \ - go/exp/regexp/syntax/parse.go \ - go/exp/regexp/syntax/perl_groups.go \ - go/exp/regexp/syntax/prog.go \ - go/exp/regexp/syntax/regexp.go \ - go/exp/regexp/syntax/simplify.go + go/exp/template/html/css.go \ + go/exp/template/html/doc.go \ + go/exp/template/html/error.go \ + go/exp/template/html/escape.go \ + go/exp/template/html/html.go \ + go/exp/template/html/js.go \ + go/exp/template/html/transition.go \ + go/exp/template/html/url.go go_go_ast_files = \ go/go/ast/ast.go \ @@ -1615,7 +1656,8 @@ go_go_build_files = \ go_go_doc_files = \ go/go/doc/comment.go \ - go/go/doc/doc.go + go/go/doc/doc.go \ + go/go/doc/example.go go_go_parser_files = \ go/go/parser/interface.go \ @@ -1631,22 +1673,9 @@ go_go_scanner_files = \ go_go_token_files = \ go/go/token/position.go \ + go/go/token/serialize.go \ go/go/token/token.go -go_go_typechecker_files = \ - go/go/typechecker/scope.go \ - go/go/typechecker/type.go \ - go/go/typechecker/typechecker.go \ - go/go/typechecker/universe.go - -go_go_types_files = \ - go/go/types/check.go \ - go/go/types/const.go \ - go/go/types/exportdata.go \ - go/go/types/gcimporter.go \ - go/go/types/types.go \ - go/go/types/universe.go - go_hash_adler32_files = \ go/hash/adler32/adler32.go @@ -1675,14 +1704,12 @@ go_http_httptest_files = \ go_http_pprof_files = \ go/http/pprof/pprof.go -go_http_spdy_files = \ - go/http/spdy/read.go \ - go/http/spdy/types.go \ - go/http/spdy/write.go - go_image_bmp_files = \ go/image/bmp/reader.go +go_image_color_files = \ + go/image/color/color.go + go_image_draw_files = \ go/image/draw/draw.go @@ -1702,6 +1729,7 @@ go_image_png_files = \ go_image_tiff_files = \ go/image/tiff/buffer.go \ + go/image/tiff/compress.go \ go/image/tiff/consts.go \ go/image/tiff/reader.go @@ -1731,6 +1759,14 @@ go_net_textproto_files = \ go/net/textproto/textproto.go \ go/net/textproto/writer.go +go_old_netchan_files = \ + go/old/netchan/common.go \ + go/old/netchan/export.go \ + go/old/netchan/import.go + +go_old_regexp_files = \ + go/old/regexp/regexp.go + go_old_template_files = \ go/old/template/doc.go \ go/old/template/execute.go \ @@ -1752,6 +1788,14 @@ go_path_filepath_files = \ go/path/filepath/path.go \ go/path/filepath/path_unix.go +go_regexp_syntax_files = \ + go/regexp/syntax/compile.go \ + go/regexp/syntax/parse.go \ + go/regexp/syntax/perl_groups.go \ + go/regexp/syntax/prog.go \ + go/regexp/syntax/regexp.go \ + go/regexp/syntax/simplify.go + go_rpc_jsonrpc_files = \ go/rpc/jsonrpc/client.go \ go/rpc/jsonrpc/server.go @@ -1878,7 +1922,6 @@ libgo_go_objs = \ cmath/cmath.lo \ crypto/crypto.lo \ csv/csv.lo \ - ebnf/ebnf.lo \ exec/exec.lo \ expvar/expvar.lo \ flag/flag.lo \ @@ -1895,7 +1938,6 @@ libgo_go_objs = \ mail/mail.lo \ mime/mime.lo \ net/net.lo \ - netchan/netchan.lo \ os/os.lo \ patch/patch.lo \ path/path.lo \ @@ -1915,7 +1957,6 @@ libgo_go_objs = \ tabwriter/tabwriter.lo \ template/template.lo \ time/time.lo \ - try/try.lo \ unicode/unicode.lo \ url/url.lo \ utf16/utf16.lo \ @@ -1932,8 +1973,8 @@ libgo_go_objs = \ container/heap.lo \ container/list.lo \ container/ring.lo \ - container/vector.lo \ crypto/aes.lo \ + crypto/bcrypt.lo \ crypto/blowfish.lo \ crypto/cast5.lo \ crypto/cipher.lo \ @@ -1976,11 +2017,16 @@ libgo_go_objs = \ encoding/git85.lo \ encoding/hex.lo \ encoding/pem.lo \ + exp/ebnf.lo \ exp/gui.lo \ exp/norm.lo \ - exp/regexp.lo \ + exp/spdy.lo \ + exp/sql.lo \ + exp/ssh.lo \ + exp/terminal.lo \ + exp/types.lo \ exp/gui/x11.lo \ - exp/regexp/syntax.lo \ + exp/sql/driver.lo \ exp/template/html.lo \ go/ast.lo \ go/build.lo \ @@ -1989,8 +2035,6 @@ libgo_go_objs = \ go/printer.lo \ go/scanner.lo \ go/token.lo \ - go/typechecker.lo \ - go/types.lo \ hash/adler32.lo \ hash/crc32.lo \ hash/crc64.lo \ @@ -1999,8 +2043,8 @@ libgo_go_objs = \ http/fcgi.lo \ http/httptest.lo \ http/pprof.lo \ - http/spdy.lo \ image/bmp.lo \ + image/color.lo \ image/draw.lo \ image/gif.lo \ image/jpeg.lo \ @@ -2012,11 +2056,14 @@ libgo_go_objs = \ mime/multipart.lo \ net/dict.lo \ net/textproto.lo \ + old/netchan.lo \ + old/regexp.lo \ old/template.lo \ $(os_lib_inotify_lo) \ os/user.lo \ os/signal.lo \ path/filepath.lo \ + regexp/syntax.lo \ rpc/jsonrpc.lo \ runtime/debug.lo \ runtime/pprof.lo \ @@ -2135,7 +2182,6 @@ TEST_PACKAGES = \ bytes/check \ cmath/check \ csv/check \ - ebnf/check \ exec/check \ expvar/check \ flag/check \ @@ -2151,7 +2197,6 @@ TEST_PACKAGES = \ mail/check \ mime/check \ net/check \ - netchan/check \ os/check \ patch/check \ path/check \ @@ -2170,7 +2215,6 @@ TEST_PACKAGES = \ tabwriter/check \ template/check \ time/check \ - try/check \ unicode/check \ url/check \ utf16/check \ @@ -2187,8 +2231,8 @@ TEST_PACKAGES = \ container/heap/check \ container/list/check \ container/ring/check \ - container/vector/check \ crypto/aes/check \ + crypto/bcrypt/check \ crypto/blowfish/check \ crypto/cast5/check \ crypto/cipher/check \ @@ -2228,9 +2272,12 @@ TEST_PACKAGES = \ encoding/git85/check \ encoding/hex/check \ encoding/pem/check \ + exp/ebnf/check \ exp/norm/check \ - exp/regexp/check \ - exp/regexp/syntax/check \ + exp/spdy/check \ + exp/sql/check \ + exp/ssh/check \ + exp/terminal/check \ exp/template/html/check \ go/ast/check \ $(go_build_check_omitted_since_it_calls_6g) \ @@ -2238,7 +2285,6 @@ TEST_PACKAGES = \ go/printer/check \ go/scanner/check \ go/token/check \ - go/typechecker/check \ $(go_types_check_omitted_since_it_calls_6g) \ hash/adler32/check \ hash/crc32/check \ @@ -2246,7 +2292,6 @@ TEST_PACKAGES = \ hash/fnv/check \ http/cgi/check \ http/fcgi/check \ - http/spdy/check \ image/draw/check \ image/jpeg/check \ image/png/check \ @@ -2256,11 +2301,14 @@ TEST_PACKAGES = \ io/ioutil/check \ mime/multipart/check \ net/textproto/check \ + old/netchan/check \ + old/regexp/check \ old/template/check \ $(os_inotify_check) \ os/user/check \ os/signal/check \ path/filepath/check \ + regexp/syntax/check \ rpc/jsonrpc/check \ sync/atomic/check \ template/parse/check \ @@ -2503,6 +2551,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sigqueue.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/string.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/thread.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/yield.Plo@am__quote@ .c.o: @am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< @@ -3183,6 +3232,13 @@ thread.lo: runtime/thread.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o thread.lo `test -f 'runtime/thread.c' || echo '$(srcdir)/'`runtime/thread.c +yield.lo: runtime/yield.c +@am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT yield.lo -MD -MP -MF $(DEPDIR)/yield.Tpo -c -o yield.lo `test -f 'runtime/yield.c' || echo '$(srcdir)/'`runtime/yield.c +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/yield.Tpo $(DEPDIR)/yield.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='runtime/yield.c' object='yield.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o yield.lo `test -f 'runtime/yield.c' || echo '$(srcdir)/'`runtime/yield.c + rtems-task-variable-add.lo: runtime/rtems-task-variable-add.c @am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT rtems-task-variable-add.lo -MD -MP -MF $(DEPDIR)/rtems-task-variable-add.Tpo -c -o rtems-task-variable-add.lo `test -f 'runtime/rtems-task-variable-add.c' || echo '$(srcdir)/'`runtime/rtems-task-variable-add.c @am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/rtems-task-variable-add.Tpo $(DEPDIR)/rtems-task-variable-add.Plo @@ -3435,26 +3491,26 @@ uninstall-toolexeclibgoexpguiDATA: test -n "$$files" || exit 0; \ echo " ( cd '$(DESTDIR)$(toolexeclibgoexpguidir)' && rm -f" $$files ")"; \ cd "$(DESTDIR)$(toolexeclibgoexpguidir)" && rm -f $$files -install-toolexeclibgoexpregexpDATA: $(toolexeclibgoexpregexp_DATA) +install-toolexeclibgoexpsqlDATA: $(toolexeclibgoexpsql_DATA) @$(NORMAL_INSTALL) - test -z "$(toolexeclibgoexpregexpdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgoexpregexpdir)" - @list='$(toolexeclibgoexpregexp_DATA)'; test -n "$(toolexeclibgoexpregexpdir)" || list=; \ + test -z "$(toolexeclibgoexpsqldir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgoexpsqldir)" + @list='$(toolexeclibgoexpsql_DATA)'; test -n "$(toolexeclibgoexpsqldir)" || list=; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ done | $(am__base_list) | \ while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(toolexeclibgoexpregexpdir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(toolexeclibgoexpregexpdir)" || exit $$?; \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(toolexeclibgoexpsqldir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(toolexeclibgoexpsqldir)" || exit $$?; \ done -uninstall-toolexeclibgoexpregexpDATA: +uninstall-toolexeclibgoexpsqlDATA: @$(NORMAL_UNINSTALL) - @list='$(toolexeclibgoexpregexp_DATA)'; test -n "$(toolexeclibgoexpregexpdir)" || list=; \ + @list='$(toolexeclibgoexpsql_DATA)'; test -n "$(toolexeclibgoexpsqldir)" || list=; \ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ test -n "$$files" || exit 0; \ - echo " ( cd '$(DESTDIR)$(toolexeclibgoexpregexpdir)' && rm -f" $$files ")"; \ - cd "$(DESTDIR)$(toolexeclibgoexpregexpdir)" && rm -f $$files + echo " ( cd '$(DESTDIR)$(toolexeclibgoexpsqldir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(toolexeclibgoexpsqldir)" && rm -f $$files install-toolexeclibgoexptemplateDATA: $(toolexeclibgoexptemplate_DATA) @$(NORMAL_INSTALL) test -z "$(toolexeclibgoexptemplatedir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgoexptemplatedir)" @@ -3695,6 +3751,26 @@ uninstall-toolexeclibgopathDATA: test -n "$$files" || exit 0; \ echo " ( cd '$(DESTDIR)$(toolexeclibgopathdir)' && rm -f" $$files ")"; \ cd "$(DESTDIR)$(toolexeclibgopathdir)" && rm -f $$files +install-toolexeclibgoregexpDATA: $(toolexeclibgoregexp_DATA) + @$(NORMAL_INSTALL) + test -z "$(toolexeclibgoregexpdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgoregexpdir)" + @list='$(toolexeclibgoregexp_DATA)'; test -n "$(toolexeclibgoregexpdir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(toolexeclibgoregexpdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(toolexeclibgoregexpdir)" || exit $$?; \ + done + +uninstall-toolexeclibgoregexpDATA: + @$(NORMAL_UNINSTALL) + @list='$(toolexeclibgoregexp_DATA)'; test -n "$(toolexeclibgoregexpdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + test -n "$$files" || exit 0; \ + echo " ( cd '$(DESTDIR)$(toolexeclibgoregexpdir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(toolexeclibgoregexpdir)" && rm -f $$files install-toolexeclibgorpcDATA: $(toolexeclibgorpc_DATA) @$(NORMAL_INSTALL) test -z "$(toolexeclibgorpcdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgorpcdir)" @@ -4112,7 +4188,7 @@ all-am: Makefile $(LIBRARIES) $(LTLIBRARIES) all-multi $(DATA) \ config.h installdirs: installdirs-recursive installdirs-am: - for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibgodir)" "$(DESTDIR)$(toolexeclibgoarchivedir)" "$(DESTDIR)$(toolexeclibgocompressdir)" "$(DESTDIR)$(toolexeclibgocontainerdir)" "$(DESTDIR)$(toolexeclibgocryptodir)" "$(DESTDIR)$(toolexeclibgocryptoopenpgpdir)" "$(DESTDIR)$(toolexeclibgocryptox509dir)" "$(DESTDIR)$(toolexeclibgodebugdir)" "$(DESTDIR)$(toolexeclibgoencodingdir)" "$(DESTDIR)$(toolexeclibgoexpdir)" "$(DESTDIR)$(toolexeclibgoexpguidir)" "$(DESTDIR)$(toolexeclibgoexpregexpdir)" "$(DESTDIR)$(toolexeclibgoexptemplatedir)" "$(DESTDIR)$(toolexeclibgogodir)" "$(DESTDIR)$(toolexeclibgohashdir)" "$(DESTDIR)$(toolexeclibgohttpdir)" "$(DESTDIR)$(toolexeclibgoimagedir)" "$(DESTDIR)$(toolexeclibgoindexdir)" "$(DESTDIR)$(toolexeclibgoiodir)" "$(DESTDIR)$(toolexeclibgomimedir)" "$(DESTDIR)$(toolexeclibgonetdir)" "$(DESTDIR)$(toolexeclibgoolddir)" "$(DESTDIR)$(toolexeclibgoosdir)" "$(DESTDIR)$(toolexeclibgopathdir)" "$(DESTDIR)$(toolexeclibgorpcdir)" "$(DESTDIR)$(toolexeclibgoruntimedir)" "$(DESTDIR)$(toolexeclibgosyncdir)" "$(DESTDIR)$(toolexeclibgotemplatedir)" "$(DESTDIR)$(toolexeclibgotestingdir)"; do \ + for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibgodir)" "$(DESTDIR)$(toolexeclibgoarchivedir)" "$(DESTDIR)$(toolexeclibgocompressdir)" "$(DESTDIR)$(toolexeclibgocontainerdir)" "$(DESTDIR)$(toolexeclibgocryptodir)" "$(DESTDIR)$(toolexeclibgocryptoopenpgpdir)" "$(DESTDIR)$(toolexeclibgocryptox509dir)" "$(DESTDIR)$(toolexeclibgodebugdir)" "$(DESTDIR)$(toolexeclibgoencodingdir)" "$(DESTDIR)$(toolexeclibgoexpdir)" "$(DESTDIR)$(toolexeclibgoexpguidir)" "$(DESTDIR)$(toolexeclibgoexpsqldir)" "$(DESTDIR)$(toolexeclibgoexptemplatedir)" "$(DESTDIR)$(toolexeclibgogodir)" "$(DESTDIR)$(toolexeclibgohashdir)" "$(DESTDIR)$(toolexeclibgohttpdir)" "$(DESTDIR)$(toolexeclibgoimagedir)" "$(DESTDIR)$(toolexeclibgoindexdir)" "$(DESTDIR)$(toolexeclibgoiodir)" "$(DESTDIR)$(toolexeclibgomimedir)" "$(DESTDIR)$(toolexeclibgonetdir)" "$(DESTDIR)$(toolexeclibgoolddir)" "$(DESTDIR)$(toolexeclibgoosdir)" "$(DESTDIR)$(toolexeclibgopathdir)" "$(DESTDIR)$(toolexeclibgoregexpdir)" "$(DESTDIR)$(toolexeclibgorpcdir)" "$(DESTDIR)$(toolexeclibgoruntimedir)" "$(DESTDIR)$(toolexeclibgosyncdir)" "$(DESTDIR)$(toolexeclibgotemplatedir)" "$(DESTDIR)$(toolexeclibgotestingdir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ done install: install-recursive @@ -4183,15 +4259,16 @@ install-exec-am: install-multi install-toolexeclibLIBRARIES \ install-toolexeclibgodebugDATA \ install-toolexeclibgoencodingDATA install-toolexeclibgoexpDATA \ install-toolexeclibgoexpguiDATA \ - install-toolexeclibgoexpregexpDATA \ + install-toolexeclibgoexpsqlDATA \ install-toolexeclibgoexptemplateDATA \ install-toolexeclibgogoDATA install-toolexeclibgohashDATA \ install-toolexeclibgohttpDATA install-toolexeclibgoimageDATA \ install-toolexeclibgoindexDATA install-toolexeclibgoioDATA \ install-toolexeclibgomimeDATA install-toolexeclibgonetDATA \ install-toolexeclibgooldDATA install-toolexeclibgoosDATA \ - install-toolexeclibgopathDATA install-toolexeclibgorpcDATA \ - install-toolexeclibgoruntimeDATA install-toolexeclibgosyncDATA \ + install-toolexeclibgopathDATA install-toolexeclibgoregexpDATA \ + install-toolexeclibgorpcDATA install-toolexeclibgoruntimeDATA \ + install-toolexeclibgosyncDATA \ install-toolexeclibgotemplateDATA \ install-toolexeclibgotestingDATA @@ -4247,7 +4324,7 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ uninstall-toolexeclibgoencodingDATA \ uninstall-toolexeclibgoexpDATA \ uninstall-toolexeclibgoexpguiDATA \ - uninstall-toolexeclibgoexpregexpDATA \ + uninstall-toolexeclibgoexpsqlDATA \ uninstall-toolexeclibgoexptemplateDATA \ uninstall-toolexeclibgogoDATA uninstall-toolexeclibgohashDATA \ uninstall-toolexeclibgohttpDATA \ @@ -4255,7 +4332,9 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ uninstall-toolexeclibgoindexDATA uninstall-toolexeclibgoioDATA \ uninstall-toolexeclibgomimeDATA uninstall-toolexeclibgonetDATA \ uninstall-toolexeclibgooldDATA uninstall-toolexeclibgoosDATA \ - uninstall-toolexeclibgopathDATA uninstall-toolexeclibgorpcDATA \ + uninstall-toolexeclibgopathDATA \ + uninstall-toolexeclibgoregexpDATA \ + uninstall-toolexeclibgorpcDATA \ uninstall-toolexeclibgoruntimeDATA \ uninstall-toolexeclibgosyncDATA \ uninstall-toolexeclibgotemplateDATA \ @@ -4290,15 +4369,16 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ install-toolexeclibgodebugDATA \ install-toolexeclibgoencodingDATA install-toolexeclibgoexpDATA \ install-toolexeclibgoexpguiDATA \ - install-toolexeclibgoexpregexpDATA \ + install-toolexeclibgoexpsqlDATA \ install-toolexeclibgoexptemplateDATA \ install-toolexeclibgogoDATA install-toolexeclibgohashDATA \ install-toolexeclibgohttpDATA install-toolexeclibgoimageDATA \ install-toolexeclibgoindexDATA install-toolexeclibgoioDATA \ install-toolexeclibgomimeDATA install-toolexeclibgonetDATA \ install-toolexeclibgooldDATA install-toolexeclibgoosDATA \ - install-toolexeclibgopathDATA install-toolexeclibgorpcDATA \ - install-toolexeclibgoruntimeDATA install-toolexeclibgosyncDATA \ + install-toolexeclibgopathDATA install-toolexeclibgoregexpDATA \ + install-toolexeclibgorpcDATA install-toolexeclibgoruntimeDATA \ + install-toolexeclibgosyncDATA \ install-toolexeclibgotemplateDATA \ install-toolexeclibgotestingDATA installcheck installcheck-am \ installdirs installdirs-am maintainer-clean \ @@ -4318,7 +4398,7 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ uninstall-toolexeclibgoencodingDATA \ uninstall-toolexeclibgoexpDATA \ uninstall-toolexeclibgoexpguiDATA \ - uninstall-toolexeclibgoexpregexpDATA \ + uninstall-toolexeclibgoexpsqlDATA \ uninstall-toolexeclibgoexptemplateDATA \ uninstall-toolexeclibgogoDATA uninstall-toolexeclibgohashDATA \ uninstall-toolexeclibgohttpDATA \ @@ -4326,7 +4406,9 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ uninstall-toolexeclibgoindexDATA uninstall-toolexeclibgoioDATA \ uninstall-toolexeclibgomimeDATA uninstall-toolexeclibgonetDATA \ uninstall-toolexeclibgooldDATA uninstall-toolexeclibgoosDATA \ - uninstall-toolexeclibgopathDATA uninstall-toolexeclibgorpcDATA \ + uninstall-toolexeclibgopathDATA \ + uninstall-toolexeclibgoregexpDATA \ + uninstall-toolexeclibgorpcDATA \ uninstall-toolexeclibgoruntimeDATA \ uninstall-toolexeclibgosyncDATA \ uninstall-toolexeclibgotemplateDATA \ @@ -4440,13 +4522,6 @@ csv/check: $(CHECK_DEPS) @$(CHECK) .PHONY: csv/check -ebnf/ebnf.lo: $(go_ebnf_files) container/vector.gox go/scanner.gox \ - go/token.gox os.gox strconv.gox unicode.gox utf8.gox - $(BUILDPACKAGE) -ebnf/check: $(CHECK_DEPS) - @$(CHECK) -.PHONY: ebnf/check - exec/exec.lo: $(go_exec_files) bytes.gox io.gox os.gox strconv.gox \ strings.gox syscall.gox $(BUILDPACKAGE) @@ -4468,7 +4543,7 @@ flag/check: $(CHECK_DEPS) .PHONY: flag/check fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox math.gox os.gox reflect.gox \ - strconv.gox strings.gox unicode.gox utf8.gox + strconv.gox strings.gox sync.gox unicode.gox utf8.gox $(BUILDPACKAGE) fmt/check: $(CHECK_DEPS) @$(CHECK) @@ -4488,8 +4563,8 @@ hash/check: $(CHECK_DEPS) @$(CHECK) .PHONY: hash/check -html/html.lo: $(go_html_files) bytes.gox io.gox os.gox strconv.gox strings.gox \ - utf8.gox +html/html.lo: $(go_html_files) bufio.gox bytes.gox fmt.gox io.gox os.gox \ + strconv.gox strings.gox utf8.gox $(BUILDPACKAGE) html/check: $(CHECK_DEPS) @$(CHECK) @@ -4506,7 +4581,8 @@ http/check: $(CHECK_DEPS) @$(CHECK) .PHONY: http/check -image/image.lo: $(go_image_files) bufio.gox io.gox os.gox strconv.gox +image/image.lo: $(go_image_files) bufio.gox image/color.gox io.gox os.gox \ + strconv.gox $(BUILDPACKAGE) image/check: $(CHECK_DEPS) @$(CHECK) @@ -4562,13 +4638,6 @@ net/check: $(CHECK_DEPS) @$(CHECK) .PHONY: net/check -netchan/netchan.lo: $(go_netchan_files) gob.gox io.gox log.gox net.gox os.gox \ - reflect.gox strconv.gox sync.gox time.gox - $(BUILDPACKAGE) -netchan/check: $(CHECK_DEPS) - @$(CHECK) -.PHONY: netchan/check - os/os.lo: $(go_os_files) runtime.gox sync.gox syscall.gox $(BUILDPACKAGE) os/check: $(CHECK_DEPS) @@ -4606,8 +4675,8 @@ reflect/check: $(CHECK_DEPS) @$(CHECK) .PHONY: reflect/check -regexp/regexp.lo: $(go_regexp_files) bytes.gox io.gox os.gox strings.gox \ - utf8.gox +regexp/regexp.lo: $(go_regexp_files) bytes.gox io.gox os.gox \ + regexp/syntax.gox strconv.gox strings.gox sync.gox utf8.gox $(BUILDPACKAGE) regexp/check: $(CHECK_DEPS) @$(CHECK) @@ -4654,7 +4723,7 @@ strconv/check: $(CHECK_DEPS) @$(CHECK) .PHONY: strconv/check -strings/strings.lo: $(go_strings_files) os.gox unicode.gox utf8.gox +strings/strings.lo: $(go_strings_files) io.gox os.gox unicode.gox utf8.gox $(BUILDPACKAGE) strings/check: $(CHECK_DEPS) @$(CHECK) @@ -4689,8 +4758,9 @@ template/check: $(CHECK_DEPS) @$(CHECK) .PHONY: template/check -testing/testing.lo: $(go_testing_files) flag.gox fmt.gox os.gox regexp.gox \ - runtime.gox runtime/pprof.gox strings.gox strconv.gox time.gox +testing/testing.lo: $(go_testing_files) bytes.gox flag.gox fmt.gox io.gox \ + os.gox regexp.gox runtime.gox runtime/pprof.gox strings.gox \ + strconv.gox time.gox $(BUILDPACKAGE) testing/check: $(CHECK_DEPS) @$(CHECK) @@ -4703,12 +4773,6 @@ time/check: $(CHECK_DEPS) @$(CHECK) .PHONY: time/check -try/try.lo: $(go_try_files) fmt.gox io.gox os.gox reflect.gox unicode.gox - $(BUILDPACKAGE) -try/check: $(CHECK_DEPS) - @$(CHECK) -.PHONY: try/check - unicode/unicode.lo: $(go_unicode_files) $(BUILDPACKAGE) unicode/check: $(CHECK_DEPS) @@ -4734,8 +4798,10 @@ utf8/check: $(CHECK_DEPS) .PHONY: utf8/check websocket/websocket.lo: $(go_websocket_files) bufio.gox bytes.gox \ - crypto/md5.gox crypto/tls.gox encoding/binary.gox fmt.gox \ - http.gox io.gox net.gox os.gox rand.gox strings.gox url.gox + crypto/md5.gox crypto/rand.gox crypto/sha1.gox crypto/tls.gox \ + encoding/base64.gox encoding/binary.gox fmt.gox http.gox \ + io.gox io/ioutil.gox json.gox net.gox os.gox rand.gox \ + strings.gox strconv.gox sync.gox url.gox $(BUILDPACKAGE) websocket/check: $(CHECK_DEPS) @$(CHECK) @@ -4824,13 +4890,6 @@ container/ring/check: $(CHECK_DEPS) @$(CHECK) .PHONY: container/ring/check -container/vector.lo: $(go_container_vector_files) - $(BUILDPACKAGE) -container/vector/check: $(CHECK_DEPS) - @$(MKDIR_P) container/vector - @$(CHECK) -.PHONY: container/vector/check - crypto/aes.lo: $(go_crypto_aes_files) os.gox strconv.gox $(BUILDPACKAGE) crypto/aes/check: $(CHECK_DEPS) @@ -4838,6 +4897,15 @@ crypto/aes/check: $(CHECK_DEPS) @$(CHECK) .PHONY: crypto/aes/check +crypto/bcrypt.lo: $(go_crypto_bcrypt_files) crypto/blowfish.gox \ + crypto/rand.gox crypto/subtle.gox encoding/base64.gox \ + fmt.gox io.gox os.gox strconv.gox + $(BUILDPACKAGE) +crypto/bcrypt/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/bcrypt + @$(CHECK) +.PHONY: crypto/bcrypt/check + crypto/blowfish.lo: $(go_crypto_blowfish_files) os.gox strconv.gox $(BUILDPACKAGE) crypto/blowfish/check: $(CHECK_DEPS) @@ -4990,12 +5058,12 @@ crypto/subtle/check: $(CHECK_DEPS) .PHONY: crypto/subtle/check crypto/tls.lo: $(go_crypto_tls_files) big.gox bytes.gox crypto.gox \ - crypto/aes.gox crypto/cipher.gox crypto/elliptic.gox \ - crypto/hmac.gox crypto/md5.gox crypto/rand.gox crypto/rc4.gox \ - crypto/rsa.gox crypto/sha1.gox crypto/subtle.gox \ - crypto/x509.gox crypto/x509/pkix.gox encoding/pem.gox \ - hash.gox io.gox io/ioutil.gox net.gox os.gox strconv.gox \ - strings.gox sync.gox time.gox + crypto/aes.gox crypto/cipher.gox crypto/des.gox \ + crypto/elliptic.gox crypto/hmac.gox crypto/md5.gox \ + crypto/rand.gox crypto/rc4.gox crypto/rsa.gox crypto/sha1.gox \ + crypto/subtle.gox crypto/x509.gox crypto/x509/pkix.gox \ + encoding/pem.gox hash.gox io.gox io/ioutil.gox net.gox os.gox \ + strconv.gox strings.gox sync.gox time.gox $(BUILDPACKAGE) crypto/tls/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/tls @@ -5170,6 +5238,14 @@ encoding/pem/check: $(CHECK_DEPS) @$(CHECK) .PHONY: encoding/pem/check +exp/ebnf.lo: $(go_exp_ebnf_files) go/scanner.gox go/token.gox os.gox \ + strconv.gox unicode.gox utf8.gox + $(BUILDPACKAGE) +exp/ebnf/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/ebnf + @$(CHECK) +.PHONY: exp/ebnf/check + exp/gui.lo: $(go_exp_gui_files) image.gox image/draw.gox os.gox $(BUILDPACKAGE) exp/gui/check: $(CHECK_DEPS) @@ -5177,20 +5253,55 @@ exp/gui/check: $(CHECK_DEPS) @$(CHECK) .PHONY: exp/gui/check -exp/norm.lo: $(go_exp_norm_files) utf8.gox +exp/norm.lo: $(go_exp_norm_files) io.gox os.gox utf8.gox $(BUILDPACKAGE) exp/norm/check: $(CHECK_DEPS) @$(MKDIR_P) exp/norm @$(CHECK) .PHONY: exp/norm/check -exp/regexp.lo: $(go_exp_regexp_files) bytes.gox exp/regexp/syntax.gox io.gox \ - os.gox strings.gox sync.gox utf8.gox +exp/spdy.lo: $(go_exp_spdy_files) bytes.gox compress/zlib.gox \ + encoding/binary.gox http.gox io.gox os.gox strings.gox + $(BUILDPACKAGE) +exp/spdy/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/spdy + @$(CHECK) +.PHONY: exp/spdy/check + +exp/sql.lo: $(go_exp_sql_files) exp/sql/driver.gox fmt.gox os.gox reflect.gox \ + runtime.gox strconv.gox sync.gox + $(BUILDPACKAGE) +exp/sql/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/sql + @$(CHECK) +.PHONY: exp/sql/check + +exp/ssh.lo: $(go_exp_ssh_files) big.gox bufio.gox bytes.gox crypto.gox \ + crypto/aes.gox crypto/cipher.gox crypto/hmac.gox \ + crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \ + crypto/subtle.gox crypto/x509.gox encoding/pem.gox hash.gox \ + io.gox net.gox os.gox reflect.gox strconv.gox sync.gox + $(BUILDPACKAGE) +exp/ssh/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/ssh + @$(CHECK) +.PHONY: exp/ssh/check + +exp/terminal.lo: $(go_exp_terminal_files) io.gox os.gox syscall.gox $(BUILDPACKAGE) -exp/regexp/check: $(CHECK_DEPS) - @$(MKDIR_P) exp/regexp +exp/terminal/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/terminal @$(CHECK) -.PHONY: exp/regexp/check +.PHONY: exp/terminal/check + +exp/types.lo: $(go_exp_types_files) big.gox bufio.gox fmt.gox go/ast.gox \ + go/token.gox io.gox os.gox path/filepath.gox runtime.gox \ + scanner.gox sort.gox strconv.gox strings.gox + $(BUILDPACKAGE) +exp/types/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/types + @$(CHECK) +.PHONY: exp/types/check exp/gui/x11.lo: $(go_exp_gui_x11_files) bufio.gox exp/gui.gox image.gox \ image/draw.gox io.gox log.gox net.gox os.gox strconv.gox \ @@ -5201,15 +5312,17 @@ exp/gui/x11/check: $(CHECK_DEPS) @$(CHECK) .PHONY: exp/gui/x11/check -exp/regexp/syntax.lo: $(go_exp_regexp_syntax_files) bytes.gox os.gox sort.gox strconv.gox strings.gox unicode.gox utf8.gox +exp/sql/driver.lo: $(go_exp_sql_driver_files) fmt.gox os.gox reflect.gox \ + strconv.gox $(BUILDPACKAGE) -exp/regexp/syntax/check: $(CHECK_DEPS) - @$(MKDIR_P) exp/regexp/syntax +exp/sql/driver/check: $(CHECK_DEPS) + @$(MKDIR_P) exp/sql/driver @$(CHECK) -.PHONY: exp/regexp/syntax/check +.PHONY: exp/sql/driver/check -exp/template/html.lo: $(go_exp_template_html_files) fmt.gox template.gox \ - template/parse.gox +exp/template/html.lo: $(go_exp_template_html_files) bytes.gox fmt.gox \ + html.gox json.gox os.gox strings.gox template.gox \ + template/parse.gox unicode.gox utf8.gox $(BUILDPACKAGE) exp/template/html/check: $(CHECK_DEPS) @$(MKDIR_P) exp/template/html @@ -5224,9 +5337,10 @@ go/ast/check: $(CHECK_DEPS) @$(CHECK) .PHONY: go/ast/check -go/build.lo: $(go_go_build_files) bytes.gox exec.gox fmt.gox go/parser.gox \ - go/token.gox log.gox os.gox path/filepath.gox regexp.gox \ - runtime.gox strconv.gox strings.gox runtime.gox +go/build.lo: $(go_go_build_files) bytes.gox exec.gox fmt.gox go/ast.gox \ + go/doc.gox go/parser.gox go/token.gox io/ioutil.gox log.gox \ + os.gox path.gox path/filepath.gox regexp.gox runtime.gox \ + sort.gox strconv.gox strings.gox runtime.gox unicode.gox $(BUILDPACKAGE) go/build/check: $(CHECK_DEPS) @$(MKDIR_P) go/build @@ -5243,7 +5357,7 @@ s-syslist: Makefile $(STAMP) $@ go/doc.lo: $(go_go_doc_files) go/ast.gox go/token.gox io.gox regexp.gox \ - sort.gox strings.gox template.gox + sort.gox strings.gox template.gox unicode.gox utf8.gox $(BUILDPACKAGE) go/doc/check: $(CHECK_DEPS) @$(MKDIR_P) go/doc @@ -5261,7 +5375,7 @@ go/parser/check: $(CHECK_DEPS) go/printer.lo: $(go_go_printer_files) bytes.gox fmt.gox go/ast.gox \ go/token.gox io.gox os.gox path/filepath.gox reflect.gox \ - runtime.gox strings.gox tabwriter.gox + strings.gox tabwriter.gox $(BUILDPACKAGE) go/printer/check: $(CHECK_DEPS) @$(MKDIR_P) go/printer @@ -5277,30 +5391,14 @@ go/scanner/check: $(CHECK_DEPS) @$(CHECK) .PHONY: go/scanner/check -go/token.lo: $(go_go_token_files) fmt.gox strconv.gox +go/token.lo: $(go_go_token_files) fmt.gox gob.gox io.gox os.gox sort.gox \ + strconv.gox sync.gox $(BUILDPACKAGE) go/token/check: $(CHECK_DEPS) @$(MKDIR_P) go/token @$(CHECK) .PHONY: go/token/check -go/typechecker.lo: $(go_go_typechecker_files) fmt.gox go/ast.gox go/token.gox \ - go/scanner.gox os.gox - $(BUILDPACKAGE) -go/typechecker/check: $(CHECK_DEPS) - @$(MKDIR_P) go/typechecker - @$(CHECK) -.PHONY: go/typechecker/check - -go/types.lo: $(go_go_types_files) big.gox bufio.gox fmt.gox go/ast.gox \ - go/token.gox io.gox os.gox path/filepath.gox runtime.gox \ - scanner.gox sort.gox strconv.gox strings.gox - $(BUILDPACKAGE) -go/types/check: $(CHECK_DEPS) - @$(MKDIR_P) go/types - @$(CHECK) -.PHONY: go/types/check - hash/adler32.lo: $(go_hash_adler32_files) hash.gox os.gox $(BUILDPACKAGE) hash/adler32/check: $(CHECK_DEPS) @@ -5364,23 +5462,21 @@ http/pprof/check: $(CHECK_DEPS) @$(CHECK) .PHONY: http/pprof/check -http/spdy.lo: $(go_http_spdy_files) bytes.gox compress/zlib.gox \ - encoding/binary.gox http.gox io.gox os.gox strconv.gox \ - strings.gox sync.gox - $(BUILDPACKAGE) -http/spdy/check: $(CHECK_DEPS) - @$(MKDIR_P) http/spdy - @$(CHECK) -.PHONY: http/spdy/check - -image/bmp.lo: $(go_image_bmp_files) image.gox io.gox os.gox +image/bmp.lo: $(go_image_bmp_files) image.gox image/color.gox io.gox os.gox $(BUILDPACKAGE) image/bmp/check: $(CHECK_DEPS) @$(MKDIR_P) image/bmp @$(CHECK) .PHONY: image/bmp/check -image/draw.lo: $(go_image_draw_files) image.gox image/ycbcr.gox +image/color.lo: $(go_image_color_files) + $(BUILDPACKAGE) +image/color/check: $(CHECK_DEPS) + @$(MKDIR_P) image/color + @$(CHECK) +.PHONY: image/color/check + +image/draw.lo: $(go_image_draw_files) image.gox image/color.gox image/ycbcr.gox $(BUILDPACKAGE) image/draw/check: $(CHECK_DEPS) @$(MKDIR_P) image/draw @@ -5388,46 +5484,48 @@ image/draw/check: $(CHECK_DEPS) .PHONY: image/draw/check image/gif.lo: $(go_image_gif_files) bufio.gox compress/lzw.gox fmt.gox \ - image.gox io.gox os.gox + image.gox image/color.gox io.gox os.gox $(BUILDPACKAGE) image/gif/check: $(CHECK_DEPS) @$(MKDIR_P) image/gif @$(CHECK) .PHONY: image/gif/check -image/jpeg.lo: $(go_image_jpeg_files) bufio.gox image.gox image/ycbcr.gox \ - io.gox os.gox +image/jpeg.lo: $(go_image_jpeg_files) bufio.gox image.gox image/color.gox \ + image/ycbcr.gox io.gox os.gox $(BUILDPACKAGE) image/jpeg/check: $(CHECK_DEPS) @$(MKDIR_P) image/jpeg @$(CHECK) .PHONY: image/jpeg/check -image/png.lo: $(go_image_png_files) bufio.gox compress/zlib.gox fmt.gox \ - hash.gox hash/crc32.gox image.gox io.gox os.gox strconv.gox +image/png.lo: $(go_image_png_files) bufio.gox compress/zlib.gox \ + encoding/binary.gox fmt.gox hash.gox hash/crc32.gox image.gox \ + image/color.gox io.gox os.gox strconv.gox $(BUILDPACKAGE) image/png/check: $(CHECK_DEPS) @$(MKDIR_P) image/png @$(CHECK) .PHONY: image/png/check -image/tiff.lo: $(go_image_tiff_files) compress/lzw.gox compress/zlib.gox \ - encoding/binary.gox image.gox io.gox io/ioutil.gox os.gox +image/tiff.lo: $(go_image_tiff_files) bufio.gox compress/lzw.gox \ + compress/zlib.gox encoding/binary.gox image.gox \ + image/color.gox io.gox io/ioutil.gox os.gox $(BUILDPACKAGE) image/tiff/check: $(CHECK_DEPS) @$(MKDIR_P) image/tiff @$(CHECK) .PHONY: image/tiff/check -image/ycbcr.lo: $(go_image_ycbcr_files) image.gox +image/ycbcr.lo: $(go_image_ycbcr_files) image.gox image/color.gox $(BUILDPACKAGE) image/ycbcr/check: $(CHECK_DEPS) @$(MKDIR_P) image/ycbcr @$(CHECK) .PHONY: image/ycbcr/check -index/suffixarray.lo: $(go_index_suffixarray_files) bytes.gox regexp.gox \ - sort.gox +index/suffixarray.lo: $(go_index_suffixarray_files) bytes.gox \ + encoding/binary.gox io.gox os.gox regexp.gox sort.gox $(BUILDPACKAGE) index/suffixarray/check: $(CHECK_DEPS) @$(MKDIR_P) index/suffixarray @@ -5456,13 +5554,30 @@ net/dict.lo: $(go_net_dict_files) net/textproto.gox os.gox strconv.gox \ $(BUILDPACKAGE) net/textproto.lo: $(go_net_textproto_files) bufio.gox bytes.gox fmt.gox \ - io.gox io/ioutil.gox net.gox os.gox strconv.gox sync.gox + io.gox io/ioutil.gox net.gox os.gox strings.gox strconv.gox \ + sync.gox $(BUILDPACKAGE) net/textproto/check: $(CHECK_DEPS) @$(MKDIR_P) net/textproto @$(CHECK) .PHONY: net/textproto/check +old/netchan.lo: $(go_old_netchan_files) gob.gox io.gox log.gox net.gox os.gox \ + reflect.gox strconv.gox sync.gox time.gox + $(BUILDPACKAGE) +old/netchan/check: $(CHECK_DEPS) + @$(MKDIR_P) old/netchan + @$(CHECK) +.PHONY: old/netchan/check + +old/regexp.lo: $(go_old_regexp_files) bytes.gox io.gox os.gox strings.gox \ + utf8.gox + $(BUILDPACKAGE) +old/regexp/check: $(CHECK_DEPS) + @$(MKDIR_P) old/regexp + @$(CHECK) +.PHONY: old/regexp/check + old/template.lo: $(go_old_template_files) bytes.gox fmt.gox io.gox \ io/ioutil.gox os.gox reflect.gox strconv.gox strings.gox \ unicode.gox utf8.gox @@ -5502,6 +5617,14 @@ path/filepath/check: $(CHECK_DEPS) @$(CHECK) .PHONY: path/filepath/check +regexp/syntax.lo: $(go_regexp_syntax_files) bytes.gox os.gox sort.gox \ + strconv.gox strings.gox unicode.gox utf8.gox + $(BUILDPACKAGE) +regexp/syntax/check: $(CHECK_DEPS) + @$(MKDIR_P) regexp/syntax + @$(CHECK) +.PHONY: regexp/syntax/check + rpc/jsonrpc.lo: $(go_rpc_jsonrpc_files) fmt.gox io.gox json.gox net.gox \ os.gox rpc.gox sync.gox $(BUILDPACKAGE) @@ -5587,8 +5710,6 @@ crypto.gox: crypto/crypto.lo $(BUILDGOX) csv.gox: csv/csv.lo $(BUILDGOX) -ebnf.gox: ebnf/ebnf.lo - $(BUILDGOX) exec.gox: exec/exec.lo $(BUILDGOX) expvar.gox: expvar/expvar.lo @@ -5621,8 +5742,6 @@ mime.gox: mime/mime.lo $(BUILDGOX) net.gox: net/net.lo $(BUILDGOX) -netchan.gox: netchan/netchan.lo - $(BUILDGOX) os.gox: os/os.lo $(BUILDGOX) patch.gox: patch/patch.lo @@ -5663,8 +5782,6 @@ testing.gox: testing/testing.lo $(BUILDGOX) time.gox: time/time.lo $(BUILDGOX) -try.gox: try/try.lo - $(BUILDGOX) unicode.gox: unicode/unicode.lo $(BUILDGOX) url.gox: url/url.lo @@ -5700,11 +5817,11 @@ container/list.gox: container/list.lo $(BUILDGOX) container/ring.gox: container/ring.lo $(BUILDGOX) -container/vector.gox: container/vector.lo - $(BUILDGOX) crypto/aes.gox: crypto/aes.lo $(BUILDGOX) +crypto/bcrypt.gox: crypto/bcrypt.lo + $(BUILDGOX) crypto/blowfish.gox: crypto/blowfish.lo $(BUILDGOX) crypto/cast5.gox: crypto/cast5.lo @@ -5794,17 +5911,27 @@ encoding/hex.gox: encoding/hex.lo encoding/pem.gox: encoding/pem.lo $(BUILDGOX) +exp/ebnf.gox: exp/ebnf.lo + $(BUILDGOX) exp/gui.gox: exp/gui.lo $(BUILDGOX) exp/norm.gox: exp/norm.lo $(BUILDGOX) -exp/regexp.gox: exp/regexp.lo +exp/spdy.gox: exp/spdy.lo + $(BUILDGOX) +exp/sql.gox: exp/sql.lo + $(BUILDGOX) +exp/ssh.gox: exp/ssh.lo + $(BUILDGOX) +exp/terminal.gox: exp/terminal.lo + $(BUILDGOX) +exp/types.gox: exp/types.lo $(BUILDGOX) exp/gui/x11.gox: exp/gui/x11.lo $(BUILDGOX) -exp/regexp/syntax.gox: exp/regexp/syntax.lo +exp/sql/driver.gox: exp/sql/driver.lo $(BUILDGOX) exp/template/html.gox: exp/template/html.lo @@ -5824,10 +5951,6 @@ go/scanner.gox: go/scanner.lo $(BUILDGOX) go/token.gox: go/token.lo $(BUILDGOX) -go/typechecker.gox: go/typechecker.lo - $(BUILDGOX) -go/types.gox: go/types.lo - $(BUILDGOX) hash/adler32.gox: hash/adler32.lo $(BUILDGOX) @@ -5846,11 +5969,11 @@ http/httptest.gox: http/httptest.lo $(BUILDGOX) http/pprof.gox: http/pprof.lo $(BUILDGOX) -http/spdy.gox: http/spdy.lo - $(BUILDGOX) image/bmp.gox: image/bmp.lo $(BUILDGOX) +image/color.gox: image/color.lo + $(BUILDGOX) image/draw.gox: image/draw.lo $(BUILDGOX) image/gif.gox: image/gif.lo @@ -5878,6 +6001,10 @@ net/dict.gox: net/dict.lo net/textproto.gox: net/textproto.lo $(BUILDGOX) +old/netchan.gox: old/netchan.lo + $(BUILDGOX) +old/regexp.gox: old/regexp.lo + $(BUILDGOX) old/template.gox: old/template.lo $(BUILDGOX) @@ -5891,6 +6018,9 @@ os/signal.gox: os/signal.lo path/filepath.gox: path/filepath.lo $(BUILDGOX) +regexp/syntax.gox: regexp/syntax.lo + $(BUILDGOX) + rpc/jsonrpc.gox: rpc/jsonrpc.lo $(BUILDGOX) diff --git a/libgo/go/archive/tar/common.go b/libgo/go/archive/tar/common.go index 5288587..6735508 100644 --- a/libgo/go/archive/tar/common.go +++ b/libgo/go/archive/tar/common.go @@ -15,36 +15,37 @@ const ( blockSize = 512 // Types - TypeReg = '0' - TypeRegA = '\x00' - TypeLink = '1' - TypeSymlink = '2' - TypeChar = '3' - TypeBlock = '4' - TypeDir = '5' - TypeFifo = '6' - TypeCont = '7' - TypeXHeader = 'x' - TypeXGlobalHeader = 'g' + TypeReg = '0' // regular file. + TypeRegA = '\x00' // regular file. + TypeLink = '1' // hard link. + TypeSymlink = '2' // symbolic link. + TypeChar = '3' // character device node. + TypeBlock = '4' // block device node. + TypeDir = '5' // directory. + TypeFifo = '6' // fifo node. + TypeCont = '7' // reserved. + TypeXHeader = 'x' // extended header. + TypeXGlobalHeader = 'g' // global extended header. ) // A Header represents a single header in a tar archive. // Some fields may not be populated. type Header struct { - Name string - Mode int64 - Uid int - Gid int - Size int64 - Mtime int64 - Typeflag byte - Linkname string - Uname string - Gname string - Devmajor int64 - Devminor int64 - Atime int64 - Ctime int64 + Name string // name of header file entry. + Mode int64 // permission and mode bits. + Uid int // user id of owner. + Gid int // group id of owner. + Size int64 // length in bytes. + Mtime int64 // modified time; seconds since epoch. + Typeflag byte // type of header entry. + Linkname string // target name of link. + Uname string // user name of owner. + Gname string // group name of owner. + Devmajor int64 // major number of character or block device. + Devminor int64 // minor number of character or block device. + Atime int64 // access time; seconds since epoch. + Ctime int64 // status change time; seconds since epoch. + } var zeroBlock = make([]byte, blockSize) diff --git a/libgo/go/archive/tar/reader.go b/libgo/go/archive/tar/reader.go index 45d95c3..12de2ad 100644 --- a/libgo/go/archive/tar/reader.go +++ b/libgo/go/archive/tar/reader.go @@ -94,7 +94,7 @@ func (tr *Reader) skipUnread() { return } } - _, tr.err = io.Copyn(ioutil.Discard, tr.r, nr) + _, tr.err = io.CopyN(ioutil.Discard, tr.r, nr) } func (tr *Reader) verifyChecksum(header []byte) bool { diff --git a/libgo/go/archive/tar/testdata/writer.tar b/libgo/go/archive/tar/testdata/writer.tar Binary files differindex 0358f91..e6d816a 100644 --- a/libgo/go/archive/tar/testdata/writer.tar +++ b/libgo/go/archive/tar/testdata/writer.tar diff --git a/libgo/go/archive/tar/writer.go b/libgo/go/archive/tar/writer.go index 8673bad..c6ce224 100644 --- a/libgo/go/archive/tar/writer.go +++ b/libgo/go/archive/tar/writer.go @@ -134,7 +134,7 @@ func (tw *Writer) WriteHeader(hdr *Header) os.Error { tw.numeric(s.next(12), hdr.Mtime) // 136:148 s.next(8) // chksum (148:156) s.next(1)[0] = hdr.Typeflag // 156:157 - s.next(100) // linkname (157:257) + tw.cString(s.next(100), hdr.Linkname) // linkname (157:257) copy(s.next(8), []byte("ustar\x0000")) // 257:265 tw.cString(s.next(32), hdr.Uname) // 265:297 tw.cString(s.next(32), hdr.Gname) // 297:329 diff --git a/libgo/go/archive/tar/writer_test.go b/libgo/go/archive/tar/writer_test.go index 838cb7e..6cc9386 100644 --- a/libgo/go/archive/tar/writer_test.go +++ b/libgo/go/archive/tar/writer_test.go @@ -24,6 +24,10 @@ type writerTest struct { } var writerTests = []*writerTest{ + // The writer test file was produced with this command: + // tar (GNU tar) 1.26 + // ln -s small.txt link.txt + // tar -b 1 --format=ustar -c -f writer.tar small.txt small2.txt link.txt &writerTest{ file: "testdata/writer.tar", entries: []*writerTestEntry{ @@ -55,6 +59,21 @@ var writerTests = []*writerTest{ }, contents: "Google.com\n", }, + &writerTestEntry{ + header: &Header{ + Name: "link.txt", + Mode: 0777, + Uid: 1000, + Gid: 1000, + Size: 0, + Mtime: 1314603082, + Typeflag: '2', + Linkname: "small.txt", + Uname: "strings", + Gname: "strings", + }, + // no contents + }, }, }, // The truncated test file was produced using these commands: diff --git a/libgo/go/archive/zip/reader.go b/libgo/go/archive/zip/reader.go index f92f929..b0a5599 100644 --- a/libgo/go/archive/zip/reader.go +++ b/libgo/go/archive/zip/reader.go @@ -238,7 +238,7 @@ func readDirectoryHeader(f *File, r io.Reader) os.Error { commentLen := int(c.Uint16(b[32:34])) // startDiskNumber := c.Uint16(b[34:36]) // Unused // internalAttributes := c.Uint16(b[36:38]) // Unused - // externalAttributes := c.Uint32(b[38:42]) // Unused + f.ExternalAttrs = c.Uint32(b[38:42]) f.headerOffset = int64(c.Uint32(b[42:46])) d := make([]byte, filenameLen+extraLen+commentLen) if _, err := io.ReadFull(r, d); err != nil { diff --git a/libgo/go/archive/zip/reader_test.go b/libgo/go/archive/zip/reader_test.go index fd5fed2..3b7b0dc 100644 --- a/libgo/go/archive/zip/reader_test.go +++ b/libgo/go/archive/zip/reader_test.go @@ -26,6 +26,7 @@ type ZipTestFile struct { Content []byte // if blank, will attempt to compare against File File string // name of file to compare to (relative to testdata/) Mtime string // modified time in format "mm-dd-yy hh:mm:ss" + Mode uint32 } // Caution: The Mtime values found for the test files should correspond to @@ -47,11 +48,13 @@ var tests = []ZipTest{ Name: "test.txt", Content: []byte("This is a test text file.\n"), Mtime: "09-05-10 12:12:02", + Mode: 0x81a4, }, { Name: "gophercolor16x16.png", File: "gophercolor16x16.png", Mtime: "09-05-10 15:52:58", + Mode: 0x81a4, }, }, }, @@ -162,6 +165,8 @@ func readTestFile(t *testing.T, ft ZipTestFile, f *File) { t.Errorf("%s: mtime=%s (%d); want %s (%d)", f.Name, time.SecondsToUTC(got), got, mtime, want) } + testFileMode(t, f, ft.Mode) + size0 := f.UncompressedSize var b bytes.Buffer @@ -203,6 +208,19 @@ func readTestFile(t *testing.T, ft ZipTestFile, f *File) { } } +func testFileMode(t *testing.T, f *File, want uint32) { + mode, err := f.Mode() + if want == 0 { + if err == nil { + t.Errorf("%s mode: got %v, want none", f.Name, mode) + } + } else if err != nil { + t.Errorf("%s mode: %s", f.Name, err) + } else if mode != want { + t.Errorf("%s mode: want 0x%x, got 0x%x", f.Name, want, mode) + } +} + func TestInvalidFiles(t *testing.T) { const size = 1024 * 70 // 70kb b := make([]byte, size) diff --git a/libgo/go/archive/zip/struct.go b/libgo/go/archive/zip/struct.go index 1d6e70f..a32de5a 100644 --- a/libgo/go/archive/zip/struct.go +++ b/libgo/go/archive/zip/struct.go @@ -28,6 +28,9 @@ const ( directoryHeaderLen = 46 // + filename + extra + comment directoryEndLen = 22 // + comment dataDescriptorLen = 12 + + // Constants for the first byte in CreatorVersion + creatorUnix = 3 ) type FileHeader struct { @@ -42,6 +45,7 @@ type FileHeader struct { CompressedSize uint32 UncompressedSize uint32 Extra []byte + ExternalAttrs uint32 // Meaning depends on CreatorVersion Comment string } @@ -89,3 +93,18 @@ func (h *FileHeader) Mtime_ns() int64 { t := msDosTimeToTime(h.ModifiedDate, h.ModifiedTime) return t.Seconds() * 1e9 } + +// Mode returns the permission and mode bits for the FileHeader. +// An error is returned in case the information is not available. +func (h *FileHeader) Mode() (mode uint32, err os.Error) { + if h.CreatorVersion>>8 == creatorUnix { + return h.ExternalAttrs >> 16, nil + } + return 0, os.NewError("file mode not available") +} + +// SetMode changes the permission and mode bits for the FileHeader. +func (h *FileHeader) SetMode(mode uint32) { + h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8 + h.ExternalAttrs = mode << 16 +} diff --git a/libgo/go/archive/zip/writer.go b/libgo/go/archive/zip/writer.go index 2065b06..3a6dc38 100644 --- a/libgo/go/archive/zip/writer.go +++ b/libgo/go/archive/zip/writer.go @@ -69,7 +69,7 @@ func (w *Writer) Close() (err os.Error) { write(w, uint16(len(h.Comment))) write(w, uint16(0)) // disk number start write(w, uint16(0)) // internal file attributes - write(w, uint32(0)) // external file attributes + write(w, h.ExternalAttrs) write(w, h.offset) writeBytes(w, []byte(h.Name)) writeBytes(w, h.Extra) @@ -115,7 +115,7 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, os.Error) { } fh.Flags |= 0x8 // we will write a data descriptor - fh.CreatorVersion = 0x14 + fh.CreatorVersion = fh.CreatorVersion&0xff00 | 0x14 fh.ReaderVersion = 0x14 fw := &fileWriter{ diff --git a/libgo/go/archive/zip/writer_test.go b/libgo/go/archive/zip/writer_test.go index eb2a80c..b562f84 100644 --- a/libgo/go/archive/zip/writer_test.go +++ b/libgo/go/archive/zip/writer_test.go @@ -13,19 +13,45 @@ import ( // TODO(adg): a more sophisticated test suite -const testString = "Rabbits, guinea pigs, gophers, marsupial rats, and quolls." +type WriteTest struct { + Name string + Data []byte + Method uint16 + Mode uint32 +} + +var writeTests = []WriteTest{ + WriteTest{ + Name: "foo", + Data: []byte("Rabbits, guinea pigs, gophers, marsupial rats, and quolls."), + Method: Store, + }, + WriteTest{ + Name: "bar", + Data: nil, // large data set in the test + Method: Deflate, + Mode: 0x81ed, + }, +} func TestWriter(t *testing.T) { largeData := make([]byte, 1<<17) for i := range largeData { largeData[i] = byte(rand.Int()) } + writeTests[1].Data = largeData + defer func() { + writeTests[1].Data = nil + }() // write a zip file buf := new(bytes.Buffer) w := NewWriter(buf) - testCreate(t, w, "foo", []byte(testString), Store) - testCreate(t, w, "bar", largeData, Deflate) + + for _, wt := range writeTests { + testCreate(t, w, &wt) + } + if err := w.Close(); err != nil { t.Fatal(err) } @@ -35,26 +61,34 @@ func TestWriter(t *testing.T) { if err != nil { t.Fatal(err) } - testReadFile(t, r.File[0], []byte(testString)) - testReadFile(t, r.File[1], largeData) + for i, wt := range writeTests { + testReadFile(t, r.File[i], &wt) + } } -func testCreate(t *testing.T, w *Writer, name string, data []byte, method uint16) { +func testCreate(t *testing.T, w *Writer, wt *WriteTest) { header := &FileHeader{ - Name: name, - Method: method, + Name: wt.Name, + Method: wt.Method, + } + if wt.Mode != 0 { + header.SetMode(wt.Mode) } f, err := w.CreateHeader(header) if err != nil { t.Fatal(err) } - _, err = f.Write(data) + _, err = f.Write(wt.Data) if err != nil { t.Fatal(err) } } -func testReadFile(t *testing.T, f *File, data []byte) { +func testReadFile(t *testing.T, f *File, wt *WriteTest) { + if f.Name != wt.Name { + t.Fatalf("File name: got %q, want %q", f.Name, wt.Name) + } + testFileMode(t, f, wt.Mode) rc, err := f.Open() if err != nil { t.Fatal("opening:", err) @@ -67,7 +101,7 @@ func testReadFile(t *testing.T, f *File, data []byte) { if err != nil { t.Fatal("closing:", err) } - if !bytes.Equal(b, data) { - t.Errorf("File contents %q, want %q", b, data) + if !bytes.Equal(b, wt.Data) { + t.Errorf("File contents %q, want %q", b, wt.Data) } } diff --git a/libgo/go/asn1/asn1.go b/libgo/go/asn1/asn1.go index 39b676b..e7bd62e 100644 --- a/libgo/go/asn1/asn1.go +++ b/libgo/go/asn1/asn1.go @@ -516,6 +516,8 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam result, err = parseIA5String(innerBytes) case tagT61String: result, err = parseT61String(innerBytes) + case tagUTF8String: + result, err = parseUTF8String(innerBytes) case tagInteger: result, err = parseInt64(innerBytes) case tagBitString: diff --git a/libgo/go/asn1/asn1_test.go b/libgo/go/asn1/asn1_test.go index 9f48f7b..1c529bd 100644 --- a/libgo/go/asn1/asn1_test.go +++ b/libgo/go/asn1/asn1_test.go @@ -206,10 +206,10 @@ type timeTest struct { } var utcTestData = []timeTest{ - {"910506164540-0700", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, 0, -7 * 60 * 60, ""}}, - {"910506164540+0730", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, 0, 7*60*60 + 30*60, ""}}, - {"910506234540Z", true, &time.Time{1991, 05, 06, 23, 45, 40, 0, 0, 0, "UTC"}}, - {"9105062345Z", true, &time.Time{1991, 05, 06, 23, 45, 0, 0, 0, 0, "UTC"}}, + {"910506164540-0700", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, -7 * 60 * 60, ""}}, + {"910506164540+0730", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, 7*60*60 + 30*60, ""}}, + {"910506234540Z", true, &time.Time{1991, 05, 06, 23, 45, 40, 0, 0, "UTC"}}, + {"9105062345Z", true, &time.Time{1991, 05, 06, 23, 45, 0, 0, 0, "UTC"}}, {"a10506234540Z", false, nil}, {"91a506234540Z", false, nil}, {"9105a6234540Z", false, nil}, @@ -235,10 +235,10 @@ func TestUTCTime(t *testing.T) { } var generalizedTimeTestData = []timeTest{ - {"20100102030405Z", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, 0, "UTC"}}, + {"20100102030405Z", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, "UTC"}}, {"20100102030405", false, nil}, - {"20100102030405+0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, 6*60*60 + 7*60, ""}}, - {"20100102030405-0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, -6*60*60 - 7*60, ""}}, + {"20100102030405+0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 6*60*60 + 7*60, ""}}, + {"20100102030405-0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, -6*60*60 - 7*60, ""}}, } func TestGeneralizedTime(t *testing.T) { @@ -475,7 +475,7 @@ var derEncodedSelfSignedCert = Certificate{ RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 3}, Value: "false.example.com"}}, RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{1, 2, 840, 113549, 1, 9, 1}, Value: "false@example.com"}}, }, - Validity: Validity{NotBefore: &time.Time{Year: 2009, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NotAfter: &time.Time{Year: 2010, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}}, + Validity: Validity{NotBefore: &time.Time{Year: 2009, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, ZoneOffset: 0, Zone: "UTC"}, NotAfter: &time.Time{Year: 2010, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, ZoneOffset: 0, Zone: "UTC"}}, Subject: RDNSequence{ RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 6}, Value: "XX"}}, RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 8}, Value: "Some-State"}}, diff --git a/libgo/go/asn1/marshal.go b/libgo/go/asn1/marshal.go index d7eb63b..6d1f78b 100644 --- a/libgo/go/asn1/marshal.go +++ b/libgo/go/asn1/marshal.go @@ -464,11 +464,15 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) if v.Type() == rawValueType { rv := v.Interface().(RawValue) - err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) - if err != nil { - return + if len(rv.FullBytes) != 0 { + _, err = out.Write(rv.FullBytes) + } else { + err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) + if err != nil { + return + } + _, err = out.Write(rv.Bytes) } - _, err = out.Write(rv.Bytes) return } diff --git a/libgo/go/big/int.go b/libgo/go/big/int.go index 701b697..9e1d1ae 100644 --- a/libgo/go/big/int.go +++ b/libgo/go/big/int.go @@ -163,7 +163,7 @@ func (z *Int) Binomial(n, k int64) *Int { // Quo sets z to the quotient x/y for y != 0 and returns z. // If y == 0, a division-by-zero run-time panic occurs. -// See QuoRem for more details. +// Quo implements truncated division (like Go); see QuoRem for more details. func (z *Int) Quo(x, y *Int) *Int { z.abs, _ = z.abs.div(nil, x.abs, y.abs) z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign @@ -172,7 +172,7 @@ func (z *Int) Quo(x, y *Int) *Int { // Rem sets z to the remainder x%y for y != 0 and returns z. // If y == 0, a division-by-zero run-time panic occurs. -// See QuoRem for more details. +// Rem implements truncated modulus (like Go); see QuoRem for more details. func (z *Int) Rem(x, y *Int) *Int { _, z.abs = nat(nil).div(z.abs, x.abs, y.abs) z.neg = len(z.abs) > 0 && x.neg // 0 has no sign @@ -198,7 +198,7 @@ func (z *Int) QuoRem(x, y, r *Int) (*Int, *Int) { // Div sets z to the quotient x/y for y != 0 and returns z. // If y == 0, a division-by-zero run-time panic occurs. -// See DivMod for more details. +// Div implements Euclidean division (unlike Go); see DivMod for more details. func (z *Int) Div(x, y *Int) *Int { y_neg := y.neg // z may be an alias for y var r Int @@ -215,7 +215,7 @@ func (z *Int) Div(x, y *Int) *Int { // Mod sets z to the modulus x%y for y != 0 and returns z. // If y == 0, a division-by-zero run-time panic occurs. -// See DivMod for more details. +// Mod implements Euclidean modulus (unlike Go); see DivMod for more details. func (z *Int) Mod(x, y *Int) *Int { y0 := y // save y if z == y || alias(z.abs, y.abs) { diff --git a/libgo/go/big/int_test.go b/libgo/go/big/int_test.go index 03446d6..b2e16921 100644 --- a/libgo/go/big/int_test.go +++ b/libgo/go/big/int_test.go @@ -301,6 +301,9 @@ func TestGetString(t *testing.T) { func TestSetString(t *testing.T) { tmp := new(Int) for i, test := range stringTests { + // initialize to a non-zero value so that issues with parsing + // 0 are detected + tmp.SetInt64(1234567890) n1, ok1 := new(Int).SetString(test.in, test.base) n2, ok2 := tmp.SetString(test.in, test.base) expected := NewInt(test.val) diff --git a/libgo/go/big/nat.go b/libgo/go/big/nat.go index be3aff2..33d6bb1 100644 --- a/libgo/go/big/nat.go +++ b/libgo/go/big/nat.go @@ -646,7 +646,7 @@ func (z nat) scan(r io.RuneScanner, base int) (nat, int, os.Error) { } } case os.EOF: - return z, 10, nil + return z.make(0), 10, nil default: return z, 10, err } diff --git a/libgo/go/big/rat.go b/libgo/go/big/rat.go index 327b9bd..f435e63 100644 --- a/libgo/go/big/rat.go +++ b/libgo/go/big/rat.go @@ -27,9 +27,13 @@ func NewRat(a, b int64) *Rat { // SetFrac sets z to a/b and returns z. func (z *Rat) SetFrac(a, b *Int) *Rat { - z.a.Set(a) z.a.neg = a.neg != b.neg - z.b = z.b.set(b.abs) + babs := b.abs + if &z.a == b || alias(z.a.abs, babs) { + babs = nat(nil).set(babs) // make a copy + } + z.a.abs = z.a.abs.set(a.abs) + z.b = z.b.set(babs) return z.norm() } diff --git a/libgo/go/big/rat_test.go b/libgo/go/big/rat_test.go index dbc5bb6..a2b9055 100644 --- a/libgo/go/big/rat_test.go +++ b/libgo/go/big/rat_test.go @@ -330,3 +330,43 @@ func TestRatGobEncoding(t *testing.T) { } } } + +func TestIssue2379(t *testing.T) { + // 1) no aliasing + q := NewRat(3, 2) + x := new(Rat) + x.SetFrac(NewInt(3), NewInt(2)) + if x.Cmp(q) != 0 { + t.Errorf("1) got %s want %s", x, q) + } + + // 2) aliasing of numerator + x = NewRat(2, 3) + x.SetFrac(NewInt(3), x.Num()) + if x.Cmp(q) != 0 { + t.Errorf("2) got %s want %s", x, q) + } + + // 3) aliasing of denominator + x = NewRat(2, 3) + x.SetFrac(x.Denom(), NewInt(2)) + if x.Cmp(q) != 0 { + t.Errorf("3) got %s want %s", x, q) + } + + // 4) aliasing of numerator and denominator + x = NewRat(2, 3) + x.SetFrac(x.Denom(), x.Num()) + if x.Cmp(q) != 0 { + t.Errorf("4) got %s want %s", x, q) + } + + // 5) numerator and denominator are the same + q = NewRat(1, 1) + x = new(Rat) + n := NewInt(7) + x.SetFrac(n, n) + if x.Cmp(q) != 0 { + t.Errorf("5) got %s want %s", x, q) + } +} diff --git a/libgo/go/bufio/bufio.go b/libgo/go/bufio/bufio.go index 727ebfdb..2ea7af3 100644 --- a/libgo/go/bufio/bufio.go +++ b/libgo/go/bufio/bufio.go @@ -54,11 +54,11 @@ type Reader struct { } // NewReaderSize creates a new Reader whose buffer has the specified size, -// which must be greater than zero. If the argument io.Reader is already a +// which must be greater than one. If the argument io.Reader is already a // Reader with large enough size, it returns the underlying Reader. // It returns the Reader and any error. func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) { - if size <= 0 { + if size <= 1 { return nil, BufSizeError(size) } // Is it already a Reader? @@ -298,6 +298,17 @@ func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) { func (b *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { line, err = b.ReadSlice('\n') if err == ErrBufferFull { + // Handle the case where "\r\n" straddles the buffer. + if len(line) > 0 && line[len(line)-1] == '\r' { + // Put the '\r' back on buf and drop it from line. + // Let the next call to ReadLine check for "\r\n". + if b.r == 0 { + // should be unreachable + panic("bufio: tried to rewind past start of buffer") + } + b.r-- + line = line[:len(line)-1] + } return line, true, nil } @@ -307,10 +318,11 @@ func (b *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { err = nil if line[len(line)-1] == '\n' { - line = line[:len(line)-1] - } - if len(line) > 0 && line[len(line)-1] == '\r' { - line = line[:len(line)-1] + drop := 1 + if len(line) > 1 && line[len(line)-2] == '\r' { + drop = 2 + } + line = line[:len(line)-drop] } return } diff --git a/libgo/go/bufio/bufio_test.go b/libgo/go/bufio/bufio_test.go index 82c73d3..38213ff 100644 --- a/libgo/go/bufio/bufio_test.go +++ b/libgo/go/bufio/bufio_test.go @@ -137,7 +137,7 @@ var bufreaders = []bufReader{ } var bufsizes = []int{ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 23, 32, 46, 64, 93, 128, 1024, 4096, } @@ -697,3 +697,71 @@ func TestLinesAfterRead(t *testing.T) { t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) } } + +type readLineResult struct { + line []byte + isPrefix bool + err os.Error +} + +var readLineNewlinesTests = []struct { + input string + bufSize int + expect []readLineResult +}{ + {"h\r\nb\r\n", 2, []readLineResult{ + {[]byte("h"), true, nil}, + {nil, false, nil}, + {[]byte("b"), true, nil}, + {nil, false, nil}, + {nil, false, os.EOF}, + }}, + {"hello\r\nworld\r\n", 6, []readLineResult{ + {[]byte("hello"), true, nil}, + {nil, false, nil}, + {[]byte("world"), true, nil}, + {nil, false, nil}, + {nil, false, os.EOF}, + }}, + {"hello\rworld\r", 6, []readLineResult{ + {[]byte("hello"), true, nil}, + {[]byte("\rworld"), true, nil}, + {[]byte("\r"), false, nil}, + {nil, false, os.EOF}, + }}, + {"h\ri\r\n\r", 2, []readLineResult{ + {[]byte("h"), true, nil}, + {[]byte("\ri"), true, nil}, + {nil, false, nil}, + {[]byte("\r"), false, nil}, + {nil, false, os.EOF}, + }}, +} + +func TestReadLineNewlines(t *testing.T) { + for _, e := range readLineNewlinesTests { + testReadLineNewlines(t, e.input, e.bufSize, e.expect) + } +} + +func testReadLineNewlines(t *testing.T, input string, bufSize int, expect []readLineResult) { + b, err := NewReaderSize(strings.NewReader(input), bufSize) + if err != nil { + t.Fatal(err) + } + for i, e := range expect { + line, isPrefix, err := b.ReadLine() + if bytes.Compare(line, e.line) != 0 { + t.Errorf("%q call %d, line == %q, want %q", input, i, line, e.line) + return + } + if isPrefix != e.isPrefix { + t.Errorf("%q call %d, isPrefix == %v, want %v", input, i, isPrefix, e.isPrefix) + return + } + if err != e.err { + t.Errorf("%q call %d, err == %v, want %v", input, i, err, e.err) + return + } + } +} diff --git a/libgo/go/bytes/buffer.go b/libgo/go/bytes/buffer.go index 5de8610..975031b 100644 --- a/libgo/go/bytes/buffer.go +++ b/libgo/go/bytes/buffer.go @@ -336,13 +336,18 @@ func (b *Buffer) ReadString(delim byte) (line string, err os.Error) { // NewBuffer creates and initializes a new Buffer using buf as its initial // contents. It is intended to prepare a Buffer to read existing data. It -// can also be used to size the internal buffer for writing. To do that, +// can also be used to size the internal buffer for writing. To do that, // buf should have the desired capacity but a length of zero. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is +// preferable to NewBuffer. In particular, passing a non-empty buf to +// NewBuffer and then writing to the Buffer will overwrite buf, not append to +// it. func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } // NewBufferString creates and initializes a new Buffer using string s as its // initial contents. It is intended to prepare a buffer to read an existing -// string. +// string. See the warnings about NewBuffer; similar issues apply here. func NewBufferString(s string) *Buffer { return &Buffer{buf: []byte(s)} } diff --git a/libgo/go/bytes/bytes.go b/libgo/go/bytes/bytes.go index 5119fce..2fb4569 100644 --- a/libgo/go/bytes/bytes.go +++ b/libgo/go/bytes/bytes.go @@ -572,13 +572,18 @@ func Runes(s []byte) []int { // non-overlapping instances of old replaced by new. // If n < 0, there is no limit on the number of replacements. func Replace(s, old, new []byte, n int) []byte { - if n == 0 { - return s // avoid allocation - } - // Compute number of replacements. - if m := Count(s, old); m == 0 { - return s // avoid allocation - } else if n <= 0 || m < n { + m := 0 + if n != 0 { + // Compute number of replacements. + m = Count(s, old) + } + if m == 0 { + // Nothing to do. Just copy. + t := make([]byte, len(s)) + copy(t, s) + return t + } + if n < 0 || m < n { n = m } @@ -603,3 +608,58 @@ func Replace(s, old, new []byte, n int) []byte { w += copy(t[w:], s[start:]) return t[0:w] } + +// EqualFold reports whether s and t, interpreted as UTF-8 strings, +// are equal under Unicode case-folding. +func EqualFold(s, t []byte) bool { + for len(s) != 0 && len(t) != 0 { + // Extract first rune from each. + var sr, tr int + if s[0] < utf8.RuneSelf { + sr, s = int(s[0]), s[1:] + } else { + r, size := utf8.DecodeRune(s) + sr, s = r, s[size:] + } + if t[0] < utf8.RuneSelf { + tr, t = int(t[0]), t[1:] + } else { + r, size := utf8.DecodeRune(t) + tr, t = r, t[size:] + } + + // If they match, keep going; if not, return false. + + // Easy case. + if tr == sr { + continue + } + + // Make sr < tr to simplify what follows. + if tr < sr { + tr, sr = sr, tr + } + // Fast check for ASCII. + if tr < utf8.RuneSelf && 'A' <= sr && sr <= 'Z' { + // ASCII, and sr is upper case. tr must be lower case. + if tr == sr+'a'-'A' { + continue + } + return false + } + + // General case. SimpleFold(x) returns the next equivalent rune > x + // or wraps around to smaller values. + r := unicode.SimpleFold(sr) + for r != sr && r < tr { + r = unicode.SimpleFold(r) + } + if r == tr { + continue + } + return false + } + + // One string is empty. Are both? + return len(s) == len(t) +} diff --git a/libgo/go/bytes/bytes_test.go b/libgo/go/bytes/bytes_test.go index 9444358..ce3f37e 100644 --- a/libgo/go/bytes/bytes_test.go +++ b/libgo/go/bytes/bytes_test.go @@ -829,9 +829,15 @@ var ReplaceTests = []ReplaceTest{ func TestReplace(t *testing.T) { for _, tt := range ReplaceTests { - if s := string(Replace([]byte(tt.in), []byte(tt.old), []byte(tt.new), tt.n)); s != tt.out { + in := append([]byte(tt.in), "<spare>"...) + in = in[:len(tt.in)] + out := Replace(in, []byte(tt.old), []byte(tt.new), tt.n) + if s := string(out); s != tt.out { t.Errorf("Replace(%q, %q, %q, %d) = %q, want %q", tt.in, tt.old, tt.new, tt.n, s, tt.out) } + if cap(in) == cap(out) && &in[:1][0] == &out[:1][0] { + t.Errorf("Replace(%q, %q, %q, %d) didn't copy", tt.in, tt.old, tt.new, tt.n) + } } } @@ -856,3 +862,31 @@ func TestTitle(t *testing.T) { } } } + +var EqualFoldTests = []struct { + s, t string + out bool +}{ + {"abc", "abc", true}, + {"ABcd", "ABcd", true}, + {"123abc", "123ABC", true}, + {"αβδ", "ΑΒΔ", true}, + {"abc", "xyz", false}, + {"abc", "XYZ", false}, + {"abcdefghijk", "abcdefghijX", false}, + {"abcdefghijk", "abcdefghij\u212A", true}, + {"abcdefghijK", "abcdefghij\u212A", true}, + {"abcdefghijkz", "abcdefghij\u212Ay", false}, + {"abcdefghijKz", "abcdefghij\u212Ay", false}, +} + +func TestEqualFold(t *testing.T) { + for _, tt := range EqualFoldTests { + if out := EqualFold([]byte(tt.s), []byte(tt.t)); out != tt.out { + t.Errorf("EqualFold(%#q, %#q) = %v, want %v", tt.s, tt.t, out, tt.out) + } + if out := EqualFold([]byte(tt.t), []byte(tt.s)); out != tt.out { + t.Errorf("EqualFold(%#q, %#q) = %v, want %v", tt.t, tt.s, out, tt.out) + } + } +} diff --git a/libgo/go/cmath/asin.go b/libgo/go/cmath/asin.go index d6a3ca4..01ce80a 100644 --- a/libgo/go/cmath/asin.go +++ b/libgo/go/cmath/asin.go @@ -50,7 +50,7 @@ import "math" // Asin returns the inverse sine of x. func Asin(x complex128) complex128 { if imag(x) == 0 { - if math.Fabs(real(x)) > 1 { + if math.Abs(real(x)) > 1 { return complex(math.Pi/2, 0) // DOMAIN error } return complex(math.Asin(real(x)), 0) @@ -67,7 +67,7 @@ func Asin(x complex128) complex128 { func Asinh(x complex128) complex128 { // TODO check range if imag(x) == 0 { - if math.Fabs(real(x)) > 1 { + if math.Abs(real(x)) > 1 { return complex(math.Pi/2, 0) // DOMAIN error } return complex(math.Asinh(real(x)), 0) diff --git a/libgo/go/cmath/sin.go b/libgo/go/cmath/sin.go index 8900ecd..486b717 100644 --- a/libgo/go/cmath/sin.go +++ b/libgo/go/cmath/sin.go @@ -122,7 +122,7 @@ func Cosh(x complex128) complex128 { // calculate sinh and cosh func sinhcosh(x float64) (sh, ch float64) { - if math.Fabs(x) <= 0.5 { + if math.Abs(x) <= 0.5 { return math.Sinh(x), math.Cosh(x) } e := math.Exp(x) diff --git a/libgo/go/cmath/sqrt.go b/libgo/go/cmath/sqrt.go index e77a9b9..4e7e805 100644 --- a/libgo/go/cmath/sqrt.go +++ b/libgo/go/cmath/sqrt.go @@ -76,7 +76,7 @@ func Sqrt(x complex128) complex128 { b := imag(x) var scale float64 // Rescale to avoid internal overflow or underflow. - if math.Fabs(a) > 4 || math.Fabs(b) > 4 { + if math.Abs(a) > 4 || math.Abs(b) > 4 { a *= 0.25 b *= 0.25 scale = 2 @@ -89,11 +89,11 @@ func Sqrt(x complex128) complex128 { var t float64 if a > 0 { t = math.Sqrt(0.5*r + 0.5*a) - r = scale * math.Fabs((0.5*b)/t) + r = scale * math.Abs((0.5*b)/t) t *= scale } else { r = math.Sqrt(0.5*r - 0.5*a) - t = scale * math.Fabs((0.5*b)/r) + t = scale * math.Abs((0.5*b)/r) r *= scale } if b < 0 { diff --git a/libgo/go/cmath/tan.go b/libgo/go/cmath/tan.go index 94b5175..67dc22a 100644 --- a/libgo/go/cmath/tan.go +++ b/libgo/go/cmath/tan.go @@ -58,7 +58,7 @@ import "math" // Tan returns the tangent of x. func Tan(x complex128) complex128 { d := math.Cos(2*real(x)) + math.Cosh(2*imag(x)) - if math.Fabs(d) < 0.25 { + if math.Abs(d) < 0.25 { d = tanSeries(x) } if d == 0 { @@ -109,8 +109,8 @@ func reducePi(x float64) float64 { // Taylor series expansion for cosh(2y) - cos(2x) func tanSeries(z complex128) float64 { const MACHEP = 1.0 / (1 << 53) - x := math.Fabs(2 * real(z)) - y := math.Fabs(2 * imag(z)) + x := math.Abs(2 * real(z)) + y := math.Abs(2 * imag(z)) x = reducePi(x) x = x * x y = y * y @@ -139,7 +139,7 @@ func tanSeries(z complex128) float64 { t = y2 - x2 t /= f d += t - if math.Fabs(t/d) <= MACHEP { + if math.Abs(t/d) <= MACHEP { break } } @@ -174,7 +174,7 @@ func tanSeries(z complex128) float64 { // Cot returns the cotangent of x. func Cot(x complex128) complex128 { d := math.Cosh(2*imag(x)) - math.Cos(2*real(x)) - if math.Fabs(d) < 0.25 { + if math.Abs(d) < 0.25 { d = tanSeries(x) } if d == 0 { diff --git a/libgo/go/container/heap/heap_test.go b/libgo/go/container/heap/heap_test.go index c5c1f76..6625e3a 100644 --- a/libgo/go/container/heap/heap_test.go +++ b/libgo/go/container/heap/heap_test.go @@ -6,32 +6,46 @@ package heap_test import ( "testing" - "container/vector" . "container/heap" ) -type myHeap struct { - // A vector.Vector implements sort.Interface except for Less, - // and it implements Push and Pop as required for heap.Interface. - vector.Vector +type myHeap []int + +func (h *myHeap) Less(i, j int) bool { + return (*h)[i] < (*h)[j] +} + +func (h *myHeap) Swap(i, j int) { + (*h)[i], (*h)[j] = (*h)[j], (*h)[i] +} + +func (h *myHeap) Len() int { + return len(*h) +} + +func (h *myHeap) Pop() (v interface{}) { + *h, v = (*h)[:h.Len()-1], (*h)[h.Len()-1] + return } -func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) } +func (h *myHeap) Push(v interface{}) { + *h = append(*h, v.(int)) +} -func (h *myHeap) verify(t *testing.T, i int) { +func (h myHeap) verify(t *testing.T, i int) { n := h.Len() j1 := 2*i + 1 j2 := 2*i + 2 if j1 < n { if h.Less(j1, i) { - t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1)) + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h[i], j1, h[j1]) return } h.verify(t, j1) } if j2 < n { if h.Less(j2, i) { - t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2)) + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h[i], j1, h[j2]) return } h.verify(t, j2) diff --git a/libgo/go/container/vector/defs.go b/libgo/go/container/vector/defs.go deleted file mode 100644 index 6d6b2ac..0000000 --- a/libgo/go/container/vector/defs.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package vector implements containers for managing sequences of elements. -// Vectors grow and shrink dynamically as necessary. -package vector - -// Vector is a container for numbered sequences of elements of type interface{}. -// A vector's length and capacity adjusts automatically as necessary. -// The zero value for Vector is an empty vector ready to use. -type Vector []interface{} - -// IntVector is a container for numbered sequences of elements of type int. -// A vector's length and capacity adjusts automatically as necessary. -// The zero value for IntVector is an empty vector ready to use. -type IntVector []int - -// StringVector is a container for numbered sequences of elements of type string. -// A vector's length and capacity adjusts automatically as necessary. -// The zero value for StringVector is an empty vector ready to use. -type StringVector []string - -// Initial underlying array size -const initialSize = 8 - -// Partial sort.Interface support - -// LessInterface provides partial support of the sort.Interface. -type LessInterface interface { - Less(y interface{}) bool -} - -// Less returns a boolean denoting whether the i'th element is less than the j'th element. -func (p *Vector) Less(i, j int) bool { return (*p)[i].(LessInterface).Less((*p)[j]) } - -// sort.Interface support - -// Less returns a boolean denoting whether the i'th element is less than the j'th element. -func (p *IntVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } - -// Less returns a boolean denoting whether the i'th element is less than the j'th element. -func (p *StringVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } diff --git a/libgo/go/container/vector/intvector.go b/libgo/go/container/vector/intvector.go deleted file mode 100644 index aa88cfe..0000000 --- a/libgo/go/container/vector/intvector.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// CAUTION: If this file is not vector.go, it was generated -// automatically from vector.go - DO NOT EDIT in that case! - -package vector - -func (p *IntVector) realloc(length, capacity int) (b []int) { - if capacity < initialSize { - capacity = initialSize - } - if capacity < length { - capacity = length - } - b = make(IntVector, length, capacity) - copy(b, *p) - *p = b - return -} - -// Insert n elements at position i. -func (p *IntVector) Expand(i, n int) { - a := *p - - // make sure we have enough space - len0 := len(a) - len1 := len0 + n - if len1 <= cap(a) { - // enough space - just expand - a = a[0:len1] - } else { - // not enough space - double capacity - capb := cap(a) * 2 - if capb < len1 { - // still not enough - use required length - capb = len1 - } - // capb >= len1 - a = p.realloc(len1, capb) - } - - // make a hole - for j := len0 - 1; j >= i; j-- { - a[j+n] = a[j] - } - - *p = a -} - -// Insert n elements at the end of a vector. -func (p *IntVector) Extend(n int) { p.Expand(len(*p), n) } - -// Resize changes the length and capacity of a vector. -// If the new length is shorter than the current length, Resize discards -// trailing elements. If the new length is longer than the current length, -// Resize adds the respective zero values for the additional elements. The capacity -// parameter is ignored unless the new length or capacity is longer than the current -// capacity. The resized vector's capacity may be larger than the requested capacity. -func (p *IntVector) Resize(length, capacity int) *IntVector { - a := *p - - if length > cap(a) || capacity > cap(a) { - // not enough space or larger capacity requested explicitly - a = p.realloc(length, capacity) - } else if length < len(a) { - // clear trailing elements - for i := range a[length:] { - var zero int - a[length+i] = zero - } - } - - *p = a[0:length] - return p -} - -// Len returns the number of elements in the vector. -// Same as len(*p). -func (p *IntVector) Len() int { return len(*p) } - -// Cap returns the capacity of the vector; that is, the -// maximum length the vector can grow without resizing. -// Same as cap(*p). -func (p *IntVector) Cap() int { return cap(*p) } - -// At returns the i'th element of the vector. -func (p *IntVector) At(i int) int { return (*p)[i] } - -// Set sets the i'th element of the vector to value x. -func (p *IntVector) Set(i int, x int) { (*p)[i] = x } - -// Last returns the element in the vector of highest index. -func (p *IntVector) Last() int { return (*p)[len(*p)-1] } - -// Copy makes a copy of the vector and returns it. -func (p *IntVector) Copy() IntVector { - arr := make(IntVector, len(*p)) - copy(arr, *p) - return arr -} - -// Insert inserts into the vector an element of value x before -// the current element at index i. -func (p *IntVector) Insert(i int, x int) { - p.Expand(i, 1) - (*p)[i] = x -} - -// Delete deletes the i'th element of the vector. The gap is closed so the old -// element at index i+1 has index i afterwards. -func (p *IntVector) Delete(i int) { - a := *p - n := len(a) - - copy(a[i:n-1], a[i+1:n]) - var zero int - a[n-1] = zero // support GC, zero out entry - *p = a[0 : n-1] -} - -// InsertVector inserts into the vector the contents of the vector -// x such that the 0th element of x appears at index i after insertion. -func (p *IntVector) InsertVector(i int, x *IntVector) { - b := *x - - p.Expand(i, len(b)) - copy((*p)[i:i+len(b)], b) -} - -// Cut deletes elements i through j-1, inclusive. -func (p *IntVector) Cut(i, j int) { - a := *p - n := len(a) - m := n - (j - i) - - copy(a[i:m], a[j:n]) - for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. - var zero int - a[k] = zero // support GC, zero out entries - } - - *p = a[0:m] -} - -// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. -// The elements are copied. The original vector is unchanged. -func (p *IntVector) Slice(i, j int) *IntVector { - var s IntVector - s.realloc(j-i, 0) // will fail in Init() if j < i - copy(s, (*p)[i:j]) - return &s -} - -// Convenience wrappers - -// Push appends x to the end of the vector. -func (p *IntVector) Push(x int) { p.Insert(len(*p), x) } - -// Pop deletes the last element of the vector. -func (p *IntVector) Pop() int { - a := *p - - i := len(a) - 1 - x := a[i] - var zero int - a[i] = zero // support GC, zero out entry - *p = a[0:i] - return x -} - -// AppendVector appends the entire vector x to the end of this vector. -func (p *IntVector) AppendVector(x *IntVector) { p.InsertVector(len(*p), x) } - -// Swap exchanges the elements at indexes i and j. -func (p *IntVector) Swap(i, j int) { - a := *p - a[i], a[j] = a[j], a[i] -} - -// Do calls function f for each element of the vector, in order. -// The behavior of Do is undefined if f changes *p. -func (p *IntVector) Do(f func(elem int)) { - for _, e := range *p { - f(e) - } -} diff --git a/libgo/go/container/vector/intvector_test.go b/libgo/go/container/vector/intvector_test.go deleted file mode 100644 index b825af9..0000000 --- a/libgo/go/container/vector/intvector_test.go +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// CAUTION: If this file is not vector_test.go, it was generated -// automatically from vector_test.go - DO NOT EDIT in that case! - -package vector - -import "testing" - -func TestIntZeroLen(t *testing.T) { - a := new(IntVector) - if a.Len() != 0 { - t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) - } - if len(*a) != 0 { - t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) - } - var b IntVector - if b.Len() != 0 { - t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) - } - if len(b) != 0 { - t.Errorf("%T: B4) expected 0, got %d", b, len(b)) - } -} - -func TestIntResize(t *testing.T) { - var a IntVector - checkSize(t, &a, 0, 0) - checkSize(t, a.Resize(0, 5), 0, 5) - checkSize(t, a.Resize(1, 0), 1, 5) - checkSize(t, a.Resize(10, 0), 10, 10) - checkSize(t, a.Resize(5, 0), 5, 10) - checkSize(t, a.Resize(3, 8), 3, 10) - checkSize(t, a.Resize(0, 100), 0, 100) - checkSize(t, a.Resize(11, 100), 11, 100) -} - -func TestIntResize2(t *testing.T) { - var a IntVector - checkSize(t, &a, 0, 0) - a.Push(int2IntValue(1)) - a.Push(int2IntValue(2)) - a.Push(int2IntValue(3)) - a.Push(int2IntValue(4)) - checkSize(t, &a, 4, 4) - checkSize(t, a.Resize(10, 0), 10, 10) - for i := 4; i < a.Len(); i++ { - if a.At(i) != intzero { - t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, intzero, a.At(i)) - } - } - for i := 4; i < len(a); i++ { - if a[i] != intzero { - t.Errorf("%T: expected a[%d] == %v; found %v", a, i, intzero, a[i]) - } - } -} - -func checkIntZero(t *testing.T, a *IntVector, i int) { - for j := 0; j < i; j++ { - if a.At(j) == intzero { - t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) - } - if (*a)[j] == intzero { - t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) - } - } - for ; i < a.Len(); i++ { - if a.At(i) != intzero { - t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, intzero, a.At(i)) - } - if (*a)[i] != intzero { - t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, intzero, (*a)[i]) - } - } -} - -func TestIntTrailingElements(t *testing.T) { - var a IntVector - for i := 0; i < 10; i++ { - a.Push(int2IntValue(i + 1)) - } - checkIntZero(t, &a, 10) - checkSize(t, &a, 10, 16) - checkSize(t, a.Resize(5, 0), 5, 16) - checkSize(t, a.Resize(10, 0), 10, 16) - checkIntZero(t, &a, 5) -} - -func TestIntAccess(t *testing.T) { - const n = 100 - var a IntVector - a.Resize(n, 0) - for i := 0; i < n; i++ { - a.Set(i, int2IntValue(val(i))) - } - for i := 0; i < n; i++ { - if elem2IntValue(a.At(i)) != int2IntValue(val(i)) { - t.Error(i) - } - } - var b IntVector - b.Resize(n, 0) - for i := 0; i < n; i++ { - b[i] = int2IntValue(val(i)) - } - for i := 0; i < n; i++ { - if elem2IntValue(b[i]) != int2IntValue(val(i)) { - t.Error(i) - } - } -} - -func TestIntInsertDeleteClear(t *testing.T) { - const n = 100 - var a IntVector - - for i := 0; i < n; i++ { - if a.Len() != i { - t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) - } - if len(a) != i { - t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) - } - a.Insert(0, int2IntValue(val(i))) - if elem2IntValue(a.Last()) != int2IntValue(val(0)) { - t.Errorf("%T: B", a) - } - } - for i := n - 1; i >= 0; i-- { - if elem2IntValue(a.Last()) != int2IntValue(val(0)) { - t.Errorf("%T: C", a) - } - if elem2IntValue(a.At(0)) != int2IntValue(val(i)) { - t.Errorf("%T: D", a) - } - if elem2IntValue(a[0]) != int2IntValue(val(i)) { - t.Errorf("%T: D2", a) - } - a.Delete(0) - if a.Len() != i { - t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) - } - if len(a) != i { - t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) - } - } - - if a.Len() != 0 { - t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) - } - if len(a) != 0 { - t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) - } - for i := 0; i < n; i++ { - a.Push(int2IntValue(val(i))) - if a.Len() != i+1 { - t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) - } - if len(a) != i+1 { - t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) - } - if elem2IntValue(a.Last()) != int2IntValue(val(i)) { - t.Errorf("%T: H", a) - } - } - a.Resize(0, 0) - if a.Len() != 0 { - t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) - } - if len(a) != 0 { - t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) - } - - const m = 5 - for j := 0; j < m; j++ { - a.Push(int2IntValue(j)) - for i := 0; i < n; i++ { - x := val(i) - a.Push(int2IntValue(x)) - if elem2IntValue(a.Pop()) != int2IntValue(x) { - t.Errorf("%T: J", a) - } - if a.Len() != j+1 { - t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) - } - if len(a) != j+1 { - t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) - } - } - } - if a.Len() != m { - t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) - } - if len(a) != m { - t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) - } -} - -func verify_sliceInt(t *testing.T, x *IntVector, elt, i, j int) { - for k := i; k < j; k++ { - if elem2IntValue(x.At(k)) != int2IntValue(elt) { - t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) - } - } - - s := x.Slice(i, j) - for k, n := 0, j-i; k < n; k++ { - if elem2IntValue(s.At(k)) != int2IntValue(elt) { - t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) - } - } -} - -func verify_patternInt(t *testing.T, x *IntVector, a, b, c int) { - n := a + b + c - if x.Len() != n { - t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) - } - if len(*x) != n { - t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) - } - verify_sliceInt(t, x, 0, 0, a) - verify_sliceInt(t, x, 1, a, a+b) - verify_sliceInt(t, x, 0, a+b, n) -} - -func make_vectorInt(elt, len int) *IntVector { - x := new(IntVector).Resize(len, 0) - for i := 0; i < len; i++ { - x.Set(i, int2IntValue(elt)) - } - return x -} - -func TestIntInsertVector(t *testing.T) { - // 1 - a := make_vectorInt(0, 0) - b := make_vectorInt(1, 10) - a.InsertVector(0, b) - verify_patternInt(t, a, 0, 10, 0) - // 2 - a = make_vectorInt(0, 10) - b = make_vectorInt(1, 0) - a.InsertVector(5, b) - verify_patternInt(t, a, 5, 0, 5) - // 3 - a = make_vectorInt(0, 10) - b = make_vectorInt(1, 3) - a.InsertVector(3, b) - verify_patternInt(t, a, 3, 3, 7) - // 4 - a = make_vectorInt(0, 10) - b = make_vectorInt(1, 1000) - a.InsertVector(8, b) - verify_patternInt(t, a, 8, 1000, 2) -} - -func TestIntDo(t *testing.T) { - const n = 25 - const salt = 17 - a := new(IntVector).Resize(n, 0) - for i := 0; i < n; i++ { - a.Set(i, int2IntValue(salt*i)) - } - count := 0 - a.Do(func(e int) { - i := intf2IntValue(e) - if i != int2IntValue(count*salt) { - t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(a), "should visit", n, "values; did visit", count) - } - - b := new(IntVector).Resize(n, 0) - for i := 0; i < n; i++ { - (*b)[i] = int2IntValue(salt * i) - } - count = 0 - b.Do(func(e int) { - i := intf2IntValue(e) - if i != int2IntValue(count*salt) { - t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(b), "b) should visit", n, "values; did visit", count) - } - - var c IntVector - c.Resize(n, 0) - for i := 0; i < n; i++ { - c[i] = int2IntValue(salt * i) - } - count = 0 - c.Do(func(e int) { - i := intf2IntValue(e) - if i != int2IntValue(count*salt) { - t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(c), "c) should visit", n, "values; did visit", count) - } - -} - -func TestIntVectorCopy(t *testing.T) { - // verify Copy() returns a copy, not simply a slice of the original vector - const Len = 10 - var src IntVector - for i := 0; i < Len; i++ { - src.Push(int2IntValue(i * i)) - } - dest := src.Copy() - for i := 0; i < Len; i++ { - src[i] = int2IntValue(-1) - v := elem2IntValue(dest[i]) - if v != int2IntValue(i*i) { - t.Error(tname(src), "expected", i*i, "got", v) - } - } -} diff --git a/libgo/go/container/vector/nogen_test.go b/libgo/go/container/vector/nogen_test.go deleted file mode 100644 index 7b6a259..0000000 --- a/libgo/go/container/vector/nogen_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package vector - -import ( - "fmt" - "sort" - "testing" -) - -var ( - zero interface{} - intzero int - strzero string -) - -func int2Value(x int) int { return x } -func int2IntValue(x int) int { return x } -func int2StrValue(x int) string { return string(x) } - -func elem2Value(x interface{}) int { return x.(int) } -func elem2IntValue(x int) int { return x } -func elem2StrValue(x string) string { return x } - -func intf2Value(x interface{}) int { return x.(int) } -func intf2IntValue(x interface{}) int { return x.(int) } -func intf2StrValue(x interface{}) string { return x.(string) } - -type VectorInterface interface { - Len() int - Cap() int -} - -func checkSize(t *testing.T, v VectorInterface, len, cap int) { - if v.Len() != len { - t.Errorf("%T expected len = %d; found %d", v, len, v.Len()) - } - if v.Cap() < cap { - t.Errorf("%T expected cap >= %d; found %d", v, cap, v.Cap()) - } -} - -func val(i int) int { return i*991 - 1234 } - -func TestSorting(t *testing.T) { - const n = 100 - - a := new(IntVector).Resize(n, 0) - for i := n - 1; i >= 0; i-- { - a.Set(i, n-1-i) - } - if sort.IsSorted(a) { - t.Error("int vector not sorted") - } - - b := new(StringVector).Resize(n, 0) - for i := n - 1; i >= 0; i-- { - b.Set(i, fmt.Sprint(n-1-i)) - } - if sort.IsSorted(b) { - t.Error("string vector not sorted") - } -} - -func tname(x interface{}) string { return fmt.Sprintf("%T: ", x) } diff --git a/libgo/go/container/vector/numbers_test.go b/libgo/go/container/vector/numbers_test.go deleted file mode 100644 index abe01a8..0000000 --- a/libgo/go/container/vector/numbers_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package vector - -import ( - "fmt" - "runtime" - "strings" - "testing" -) - -const memTestN = 1000000 - -func s(n uint64) string { - str := fmt.Sprintf("%d", n) - lens := len(str) - a := make([]string, (lens+2)/3) - start := lens - for i := range a { - start -= 3 - if start < 0 { - start = 0 - } - a[len(a)-i-1] = str[start:lens] - lens -= 3 - } - return strings.Join(a, " ") -} - -func TestVectorNums(t *testing.T) { - if testing.Short() { - return - } - var v Vector - c := int(0) - runtime.GC() - m0 := runtime.MemStats - v.Resize(memTestN, memTestN) - for i := 0; i < memTestN; i++ { - v.Set(i, c) - } - runtime.GC() - m := runtime.MemStats - v.Resize(0, 0) - runtime.GC() - n := m.Alloc - m0.Alloc - t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) -} - -func TestIntVectorNums(t *testing.T) { - if testing.Short() { - return - } - var v IntVector - c := int(0) - runtime.GC() - m0 := runtime.MemStats - v.Resize(memTestN, memTestN) - for i := 0; i < memTestN; i++ { - v.Set(i, c) - } - runtime.GC() - m := runtime.MemStats - v.Resize(0, 0) - runtime.GC() - n := m.Alloc - m0.Alloc - t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) -} - -func TestStringVectorNums(t *testing.T) { - if testing.Short() { - return - } - var v StringVector - c := "" - runtime.GC() - m0 := runtime.MemStats - v.Resize(memTestN, memTestN) - for i := 0; i < memTestN; i++ { - v.Set(i, c) - } - runtime.GC() - m := runtime.MemStats - v.Resize(0, 0) - runtime.GC() - n := m.Alloc - m0.Alloc - t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) -} - -func BenchmarkVectorNums(b *testing.B) { - c := int(0) - var v Vector - b.StopTimer() - runtime.GC() - b.StartTimer() - for i := 0; i < b.N; i++ { - v.Push(c) - } -} - -func BenchmarkIntVectorNums(b *testing.B) { - c := int(0) - var v IntVector - b.StopTimer() - runtime.GC() - b.StartTimer() - for i := 0; i < b.N; i++ { - v.Push(c) - } -} - -func BenchmarkStringVectorNums(b *testing.B) { - c := "" - var v StringVector - b.StopTimer() - runtime.GC() - b.StartTimer() - for i := 0; i < b.N; i++ { - v.Push(c) - } -} diff --git a/libgo/go/container/vector/stringvector.go b/libgo/go/container/vector/stringvector.go deleted file mode 100644 index dc81f06..0000000 --- a/libgo/go/container/vector/stringvector.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// CAUTION: If this file is not vector.go, it was generated -// automatically from vector.go - DO NOT EDIT in that case! - -package vector - -func (p *StringVector) realloc(length, capacity int) (b []string) { - if capacity < initialSize { - capacity = initialSize - } - if capacity < length { - capacity = length - } - b = make(StringVector, length, capacity) - copy(b, *p) - *p = b - return -} - -// Insert n elements at position i. -func (p *StringVector) Expand(i, n int) { - a := *p - - // make sure we have enough space - len0 := len(a) - len1 := len0 + n - if len1 <= cap(a) { - // enough space - just expand - a = a[0:len1] - } else { - // not enough space - double capacity - capb := cap(a) * 2 - if capb < len1 { - // still not enough - use required length - capb = len1 - } - // capb >= len1 - a = p.realloc(len1, capb) - } - - // make a hole - for j := len0 - 1; j >= i; j-- { - a[j+n] = a[j] - } - - *p = a -} - -// Insert n elements at the end of a vector. -func (p *StringVector) Extend(n int) { p.Expand(len(*p), n) } - -// Resize changes the length and capacity of a vector. -// If the new length is shorter than the current length, Resize discards -// trailing elements. If the new length is longer than the current length, -// Resize adds the respective zero values for the additional elements. The capacity -// parameter is ignored unless the new length or capacity is longer than the current -// capacity. The resized vector's capacity may be larger than the requested capacity. -func (p *StringVector) Resize(length, capacity int) *StringVector { - a := *p - - if length > cap(a) || capacity > cap(a) { - // not enough space or larger capacity requested explicitly - a = p.realloc(length, capacity) - } else if length < len(a) { - // clear trailing elements - for i := range a[length:] { - var zero string - a[length+i] = zero - } - } - - *p = a[0:length] - return p -} - -// Len returns the number of elements in the vector. -// Same as len(*p). -func (p *StringVector) Len() int { return len(*p) } - -// Cap returns the capacity of the vector; that is, the -// maximum length the vector can grow without resizing. -// Same as cap(*p). -func (p *StringVector) Cap() int { return cap(*p) } - -// At returns the i'th element of the vector. -func (p *StringVector) At(i int) string { return (*p)[i] } - -// Set sets the i'th element of the vector to value x. -func (p *StringVector) Set(i int, x string) { (*p)[i] = x } - -// Last returns the element in the vector of highest index. -func (p *StringVector) Last() string { return (*p)[len(*p)-1] } - -// Copy makes a copy of the vector and returns it. -func (p *StringVector) Copy() StringVector { - arr := make(StringVector, len(*p)) - copy(arr, *p) - return arr -} - -// Insert inserts into the vector an element of value x before -// the current element at index i. -func (p *StringVector) Insert(i int, x string) { - p.Expand(i, 1) - (*p)[i] = x -} - -// Delete deletes the i'th element of the vector. The gap is closed so the old -// element at index i+1 has index i afterwards. -func (p *StringVector) Delete(i int) { - a := *p - n := len(a) - - copy(a[i:n-1], a[i+1:n]) - var zero string - a[n-1] = zero // support GC, zero out entry - *p = a[0 : n-1] -} - -// InsertVector inserts into the vector the contents of the vector -// x such that the 0th element of x appears at index i after insertion. -func (p *StringVector) InsertVector(i int, x *StringVector) { - b := *x - - p.Expand(i, len(b)) - copy((*p)[i:i+len(b)], b) -} - -// Cut deletes elements i through j-1, inclusive. -func (p *StringVector) Cut(i, j int) { - a := *p - n := len(a) - m := n - (j - i) - - copy(a[i:m], a[j:n]) - for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. - var zero string - a[k] = zero // support GC, zero out entries - } - - *p = a[0:m] -} - -// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. -// The elements are copied. The original vector is unchanged. -func (p *StringVector) Slice(i, j int) *StringVector { - var s StringVector - s.realloc(j-i, 0) // will fail in Init() if j < i - copy(s, (*p)[i:j]) - return &s -} - -// Convenience wrappers - -// Push appends x to the end of the vector. -func (p *StringVector) Push(x string) { p.Insert(len(*p), x) } - -// Pop deletes the last element of the vector. -func (p *StringVector) Pop() string { - a := *p - - i := len(a) - 1 - x := a[i] - var zero string - a[i] = zero // support GC, zero out entry - *p = a[0:i] - return x -} - -// AppendVector appends the entire vector x to the end of this vector. -func (p *StringVector) AppendVector(x *StringVector) { p.InsertVector(len(*p), x) } - -// Swap exchanges the elements at indexes i and j. -func (p *StringVector) Swap(i, j int) { - a := *p - a[i], a[j] = a[j], a[i] -} - -// Do calls function f for each element of the vector, in order. -// The behavior of Do is undefined if f changes *p. -func (p *StringVector) Do(f func(elem string)) { - for _, e := range *p { - f(e) - } -} diff --git a/libgo/go/container/vector/stringvector_test.go b/libgo/go/container/vector/stringvector_test.go deleted file mode 100644 index c75676f..0000000 --- a/libgo/go/container/vector/stringvector_test.go +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// CAUTION: If this file is not vector_test.go, it was generated -// automatically from vector_test.go - DO NOT EDIT in that case! - -package vector - -import "testing" - -func TestStrZeroLen(t *testing.T) { - a := new(StringVector) - if a.Len() != 0 { - t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) - } - if len(*a) != 0 { - t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) - } - var b StringVector - if b.Len() != 0 { - t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) - } - if len(b) != 0 { - t.Errorf("%T: B4) expected 0, got %d", b, len(b)) - } -} - -func TestStrResize(t *testing.T) { - var a StringVector - checkSize(t, &a, 0, 0) - checkSize(t, a.Resize(0, 5), 0, 5) - checkSize(t, a.Resize(1, 0), 1, 5) - checkSize(t, a.Resize(10, 0), 10, 10) - checkSize(t, a.Resize(5, 0), 5, 10) - checkSize(t, a.Resize(3, 8), 3, 10) - checkSize(t, a.Resize(0, 100), 0, 100) - checkSize(t, a.Resize(11, 100), 11, 100) -} - -func TestStrResize2(t *testing.T) { - var a StringVector - checkSize(t, &a, 0, 0) - a.Push(int2StrValue(1)) - a.Push(int2StrValue(2)) - a.Push(int2StrValue(3)) - a.Push(int2StrValue(4)) - checkSize(t, &a, 4, 4) - checkSize(t, a.Resize(10, 0), 10, 10) - for i := 4; i < a.Len(); i++ { - if a.At(i) != strzero { - t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, strzero, a.At(i)) - } - } - for i := 4; i < len(a); i++ { - if a[i] != strzero { - t.Errorf("%T: expected a[%d] == %v; found %v", a, i, strzero, a[i]) - } - } -} - -func checkStrZero(t *testing.T, a *StringVector, i int) { - for j := 0; j < i; j++ { - if a.At(j) == strzero { - t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) - } - if (*a)[j] == strzero { - t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) - } - } - for ; i < a.Len(); i++ { - if a.At(i) != strzero { - t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, strzero, a.At(i)) - } - if (*a)[i] != strzero { - t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, strzero, (*a)[i]) - } - } -} - -func TestStrTrailingElements(t *testing.T) { - var a StringVector - for i := 0; i < 10; i++ { - a.Push(int2StrValue(i + 1)) - } - checkStrZero(t, &a, 10) - checkSize(t, &a, 10, 16) - checkSize(t, a.Resize(5, 0), 5, 16) - checkSize(t, a.Resize(10, 0), 10, 16) - checkStrZero(t, &a, 5) -} - -func TestStrAccess(t *testing.T) { - const n = 100 - var a StringVector - a.Resize(n, 0) - for i := 0; i < n; i++ { - a.Set(i, int2StrValue(val(i))) - } - for i := 0; i < n; i++ { - if elem2StrValue(a.At(i)) != int2StrValue(val(i)) { - t.Error(i) - } - } - var b StringVector - b.Resize(n, 0) - for i := 0; i < n; i++ { - b[i] = int2StrValue(val(i)) - } - for i := 0; i < n; i++ { - if elem2StrValue(b[i]) != int2StrValue(val(i)) { - t.Error(i) - } - } -} - -func TestStrInsertDeleteClear(t *testing.T) { - const n = 100 - var a StringVector - - for i := 0; i < n; i++ { - if a.Len() != i { - t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) - } - if len(a) != i { - t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) - } - a.Insert(0, int2StrValue(val(i))) - if elem2StrValue(a.Last()) != int2StrValue(val(0)) { - t.Errorf("%T: B", a) - } - } - for i := n - 1; i >= 0; i-- { - if elem2StrValue(a.Last()) != int2StrValue(val(0)) { - t.Errorf("%T: C", a) - } - if elem2StrValue(a.At(0)) != int2StrValue(val(i)) { - t.Errorf("%T: D", a) - } - if elem2StrValue(a[0]) != int2StrValue(val(i)) { - t.Errorf("%T: D2", a) - } - a.Delete(0) - if a.Len() != i { - t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) - } - if len(a) != i { - t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) - } - } - - if a.Len() != 0 { - t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) - } - if len(a) != 0 { - t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) - } - for i := 0; i < n; i++ { - a.Push(int2StrValue(val(i))) - if a.Len() != i+1 { - t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) - } - if len(a) != i+1 { - t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) - } - if elem2StrValue(a.Last()) != int2StrValue(val(i)) { - t.Errorf("%T: H", a) - } - } - a.Resize(0, 0) - if a.Len() != 0 { - t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) - } - if len(a) != 0 { - t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) - } - - const m = 5 - for j := 0; j < m; j++ { - a.Push(int2StrValue(j)) - for i := 0; i < n; i++ { - x := val(i) - a.Push(int2StrValue(x)) - if elem2StrValue(a.Pop()) != int2StrValue(x) { - t.Errorf("%T: J", a) - } - if a.Len() != j+1 { - t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) - } - if len(a) != j+1 { - t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) - } - } - } - if a.Len() != m { - t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) - } - if len(a) != m { - t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) - } -} - -func verify_sliceStr(t *testing.T, x *StringVector, elt, i, j int) { - for k := i; k < j; k++ { - if elem2StrValue(x.At(k)) != int2StrValue(elt) { - t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) - } - } - - s := x.Slice(i, j) - for k, n := 0, j-i; k < n; k++ { - if elem2StrValue(s.At(k)) != int2StrValue(elt) { - t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) - } - } -} - -func verify_patternStr(t *testing.T, x *StringVector, a, b, c int) { - n := a + b + c - if x.Len() != n { - t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) - } - if len(*x) != n { - t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) - } - verify_sliceStr(t, x, 0, 0, a) - verify_sliceStr(t, x, 1, a, a+b) - verify_sliceStr(t, x, 0, a+b, n) -} - -func make_vectorStr(elt, len int) *StringVector { - x := new(StringVector).Resize(len, 0) - for i := 0; i < len; i++ { - x.Set(i, int2StrValue(elt)) - } - return x -} - -func TestStrInsertVector(t *testing.T) { - // 1 - a := make_vectorStr(0, 0) - b := make_vectorStr(1, 10) - a.InsertVector(0, b) - verify_patternStr(t, a, 0, 10, 0) - // 2 - a = make_vectorStr(0, 10) - b = make_vectorStr(1, 0) - a.InsertVector(5, b) - verify_patternStr(t, a, 5, 0, 5) - // 3 - a = make_vectorStr(0, 10) - b = make_vectorStr(1, 3) - a.InsertVector(3, b) - verify_patternStr(t, a, 3, 3, 7) - // 4 - a = make_vectorStr(0, 10) - b = make_vectorStr(1, 1000) - a.InsertVector(8, b) - verify_patternStr(t, a, 8, 1000, 2) -} - -func TestStrDo(t *testing.T) { - const n = 25 - const salt = 17 - a := new(StringVector).Resize(n, 0) - for i := 0; i < n; i++ { - a.Set(i, int2StrValue(salt*i)) - } - count := 0 - a.Do(func(e string) { - i := intf2StrValue(e) - if i != int2StrValue(count*salt) { - t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(a), "should visit", n, "values; did visit", count) - } - - b := new(StringVector).Resize(n, 0) - for i := 0; i < n; i++ { - (*b)[i] = int2StrValue(salt * i) - } - count = 0 - b.Do(func(e string) { - i := intf2StrValue(e) - if i != int2StrValue(count*salt) { - t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(b), "b) should visit", n, "values; did visit", count) - } - - var c StringVector - c.Resize(n, 0) - for i := 0; i < n; i++ { - c[i] = int2StrValue(salt * i) - } - count = 0 - c.Do(func(e string) { - i := intf2StrValue(e) - if i != int2StrValue(count*salt) { - t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(c), "c) should visit", n, "values; did visit", count) - } - -} - -func TestStrVectorCopy(t *testing.T) { - // verify Copy() returns a copy, not simply a slice of the original vector - const Len = 10 - var src StringVector - for i := 0; i < Len; i++ { - src.Push(int2StrValue(i * i)) - } - dest := src.Copy() - for i := 0; i < Len; i++ { - src[i] = int2StrValue(-1) - v := elem2StrValue(dest[i]) - if v != int2StrValue(i*i) { - t.Error(tname(src), "expected", i*i, "got", v) - } - } -} diff --git a/libgo/go/container/vector/vector.go b/libgo/go/container/vector/vector.go deleted file mode 100644 index 8470ec0..0000000 --- a/libgo/go/container/vector/vector.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// CAUTION: If this file is not vector.go, it was generated -// automatically from vector.go - DO NOT EDIT in that case! - -package vector - -func (p *Vector) realloc(length, capacity int) (b []interface{}) { - if capacity < initialSize { - capacity = initialSize - } - if capacity < length { - capacity = length - } - b = make(Vector, length, capacity) - copy(b, *p) - *p = b - return -} - -// Insert n elements at position i. -func (p *Vector) Expand(i, n int) { - a := *p - - // make sure we have enough space - len0 := len(a) - len1 := len0 + n - if len1 <= cap(a) { - // enough space - just expand - a = a[0:len1] - } else { - // not enough space - double capacity - capb := cap(a) * 2 - if capb < len1 { - // still not enough - use required length - capb = len1 - } - // capb >= len1 - a = p.realloc(len1, capb) - } - - // make a hole - for j := len0 - 1; j >= i; j-- { - a[j+n] = a[j] - } - - *p = a -} - -// Insert n elements at the end of a vector. -func (p *Vector) Extend(n int) { p.Expand(len(*p), n) } - -// Resize changes the length and capacity of a vector. -// If the new length is shorter than the current length, Resize discards -// trailing elements. If the new length is longer than the current length, -// Resize adds the respective zero values for the additional elements. The capacity -// parameter is ignored unless the new length or capacity is longer than the current -// capacity. The resized vector's capacity may be larger than the requested capacity. -func (p *Vector) Resize(length, capacity int) *Vector { - a := *p - - if length > cap(a) || capacity > cap(a) { - // not enough space or larger capacity requested explicitly - a = p.realloc(length, capacity) - } else if length < len(a) { - // clear trailing elements - for i := range a[length:] { - var zero interface{} - a[length+i] = zero - } - } - - *p = a[0:length] - return p -} - -// Len returns the number of elements in the vector. -// Same as len(*p). -func (p *Vector) Len() int { return len(*p) } - -// Cap returns the capacity of the vector; that is, the -// maximum length the vector can grow without resizing. -// Same as cap(*p). -func (p *Vector) Cap() int { return cap(*p) } - -// At returns the i'th element of the vector. -func (p *Vector) At(i int) interface{} { return (*p)[i] } - -// Set sets the i'th element of the vector to value x. -func (p *Vector) Set(i int, x interface{}) { (*p)[i] = x } - -// Last returns the element in the vector of highest index. -func (p *Vector) Last() interface{} { return (*p)[len(*p)-1] } - -// Copy makes a copy of the vector and returns it. -func (p *Vector) Copy() Vector { - arr := make(Vector, len(*p)) - copy(arr, *p) - return arr -} - -// Insert inserts into the vector an element of value x before -// the current element at index i. -func (p *Vector) Insert(i int, x interface{}) { - p.Expand(i, 1) - (*p)[i] = x -} - -// Delete deletes the i'th element of the vector. The gap is closed so the old -// element at index i+1 has index i afterwards. -func (p *Vector) Delete(i int) { - a := *p - n := len(a) - - copy(a[i:n-1], a[i+1:n]) - var zero interface{} - a[n-1] = zero // support GC, zero out entry - *p = a[0 : n-1] -} - -// InsertVector inserts into the vector the contents of the vector -// x such that the 0th element of x appears at index i after insertion. -func (p *Vector) InsertVector(i int, x *Vector) { - b := *x - - p.Expand(i, len(b)) - copy((*p)[i:i+len(b)], b) -} - -// Cut deletes elements i through j-1, inclusive. -func (p *Vector) Cut(i, j int) { - a := *p - n := len(a) - m := n - (j - i) - - copy(a[i:m], a[j:n]) - for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. - var zero interface{} - a[k] = zero // support GC, zero out entries - } - - *p = a[0:m] -} - -// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. -// The elements are copied. The original vector is unchanged. -func (p *Vector) Slice(i, j int) *Vector { - var s Vector - s.realloc(j-i, 0) // will fail in Init() if j < i - copy(s, (*p)[i:j]) - return &s -} - -// Convenience wrappers - -// Push appends x to the end of the vector. -func (p *Vector) Push(x interface{}) { p.Insert(len(*p), x) } - -// Pop deletes the last element of the vector. -func (p *Vector) Pop() interface{} { - a := *p - - i := len(a) - 1 - x := a[i] - var zero interface{} - a[i] = zero // support GC, zero out entry - *p = a[0:i] - return x -} - -// AppendVector appends the entire vector x to the end of this vector. -func (p *Vector) AppendVector(x *Vector) { p.InsertVector(len(*p), x) } - -// Swap exchanges the elements at indexes i and j. -func (p *Vector) Swap(i, j int) { - a := *p - a[i], a[j] = a[j], a[i] -} - -// Do calls function f for each element of the vector, in order. -// The behavior of Do is undefined if f changes *p. -func (p *Vector) Do(f func(elem interface{})) { - for _, e := range *p { - f(e) - } -} diff --git a/libgo/go/container/vector/vector_test.go b/libgo/go/container/vector/vector_test.go deleted file mode 100644 index a7f47b8..0000000 --- a/libgo/go/container/vector/vector_test.go +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// CAUTION: If this file is not vector_test.go, it was generated -// automatically from vector_test.go - DO NOT EDIT in that case! - -package vector - -import "testing" - -func TestZeroLen(t *testing.T) { - a := new(Vector) - if a.Len() != 0 { - t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) - } - if len(*a) != 0 { - t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) - } - var b Vector - if b.Len() != 0 { - t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) - } - if len(b) != 0 { - t.Errorf("%T: B4) expected 0, got %d", b, len(b)) - } -} - -func TestResize(t *testing.T) { - var a Vector - checkSize(t, &a, 0, 0) - checkSize(t, a.Resize(0, 5), 0, 5) - checkSize(t, a.Resize(1, 0), 1, 5) - checkSize(t, a.Resize(10, 0), 10, 10) - checkSize(t, a.Resize(5, 0), 5, 10) - checkSize(t, a.Resize(3, 8), 3, 10) - checkSize(t, a.Resize(0, 100), 0, 100) - checkSize(t, a.Resize(11, 100), 11, 100) -} - -func TestResize2(t *testing.T) { - var a Vector - checkSize(t, &a, 0, 0) - a.Push(int2Value(1)) - a.Push(int2Value(2)) - a.Push(int2Value(3)) - a.Push(int2Value(4)) - checkSize(t, &a, 4, 4) - checkSize(t, a.Resize(10, 0), 10, 10) - for i := 4; i < a.Len(); i++ { - if a.At(i) != zero { - t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, zero, a.At(i)) - } - } - for i := 4; i < len(a); i++ { - if a[i] != zero { - t.Errorf("%T: expected a[%d] == %v; found %v", a, i, zero, a[i]) - } - } -} - -func checkZero(t *testing.T, a *Vector, i int) { - for j := 0; j < i; j++ { - if a.At(j) == zero { - t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) - } - if (*a)[j] == zero { - t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) - } - } - for ; i < a.Len(); i++ { - if a.At(i) != zero { - t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, zero, a.At(i)) - } - if (*a)[i] != zero { - t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, zero, (*a)[i]) - } - } -} - -func TestTrailingElements(t *testing.T) { - var a Vector - for i := 0; i < 10; i++ { - a.Push(int2Value(i + 1)) - } - checkZero(t, &a, 10) - checkSize(t, &a, 10, 16) - checkSize(t, a.Resize(5, 0), 5, 16) - checkSize(t, a.Resize(10, 0), 10, 16) - checkZero(t, &a, 5) -} - -func TestAccess(t *testing.T) { - const n = 100 - var a Vector - a.Resize(n, 0) - for i := 0; i < n; i++ { - a.Set(i, int2Value(val(i))) - } - for i := 0; i < n; i++ { - if elem2Value(a.At(i)) != int2Value(val(i)) { - t.Error(i) - } - } - var b Vector - b.Resize(n, 0) - for i := 0; i < n; i++ { - b[i] = int2Value(val(i)) - } - for i := 0; i < n; i++ { - if elem2Value(b[i]) != int2Value(val(i)) { - t.Error(i) - } - } -} - -func TestInsertDeleteClear(t *testing.T) { - const n = 100 - var a Vector - - for i := 0; i < n; i++ { - if a.Len() != i { - t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) - } - if len(a) != i { - t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) - } - a.Insert(0, int2Value(val(i))) - if elem2Value(a.Last()) != int2Value(val(0)) { - t.Errorf("%T: B", a) - } - } - for i := n - 1; i >= 0; i-- { - if elem2Value(a.Last()) != int2Value(val(0)) { - t.Errorf("%T: C", a) - } - if elem2Value(a.At(0)) != int2Value(val(i)) { - t.Errorf("%T: D", a) - } - if elem2Value(a[0]) != int2Value(val(i)) { - t.Errorf("%T: D2", a) - } - a.Delete(0) - if a.Len() != i { - t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) - } - if len(a) != i { - t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) - } - } - - if a.Len() != 0 { - t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) - } - if len(a) != 0 { - t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) - } - for i := 0; i < n; i++ { - a.Push(int2Value(val(i))) - if a.Len() != i+1 { - t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) - } - if len(a) != i+1 { - t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) - } - if elem2Value(a.Last()) != int2Value(val(i)) { - t.Errorf("%T: H", a) - } - } - a.Resize(0, 0) - if a.Len() != 0 { - t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) - } - if len(a) != 0 { - t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) - } - - const m = 5 - for j := 0; j < m; j++ { - a.Push(int2Value(j)) - for i := 0; i < n; i++ { - x := val(i) - a.Push(int2Value(x)) - if elem2Value(a.Pop()) != int2Value(x) { - t.Errorf("%T: J", a) - } - if a.Len() != j+1 { - t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) - } - if len(a) != j+1 { - t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) - } - } - } - if a.Len() != m { - t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) - } - if len(a) != m { - t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) - } -} - -func verify_slice(t *testing.T, x *Vector, elt, i, j int) { - for k := i; k < j; k++ { - if elem2Value(x.At(k)) != int2Value(elt) { - t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) - } - } - - s := x.Slice(i, j) - for k, n := 0, j-i; k < n; k++ { - if elem2Value(s.At(k)) != int2Value(elt) { - t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) - } - } -} - -func verify_pattern(t *testing.T, x *Vector, a, b, c int) { - n := a + b + c - if x.Len() != n { - t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) - } - if len(*x) != n { - t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) - } - verify_slice(t, x, 0, 0, a) - verify_slice(t, x, 1, a, a+b) - verify_slice(t, x, 0, a+b, n) -} - -func make_vector(elt, len int) *Vector { - x := new(Vector).Resize(len, 0) - for i := 0; i < len; i++ { - x.Set(i, int2Value(elt)) - } - return x -} - -func TestInsertVector(t *testing.T) { - // 1 - a := make_vector(0, 0) - b := make_vector(1, 10) - a.InsertVector(0, b) - verify_pattern(t, a, 0, 10, 0) - // 2 - a = make_vector(0, 10) - b = make_vector(1, 0) - a.InsertVector(5, b) - verify_pattern(t, a, 5, 0, 5) - // 3 - a = make_vector(0, 10) - b = make_vector(1, 3) - a.InsertVector(3, b) - verify_pattern(t, a, 3, 3, 7) - // 4 - a = make_vector(0, 10) - b = make_vector(1, 1000) - a.InsertVector(8, b) - verify_pattern(t, a, 8, 1000, 2) -} - -func TestDo(t *testing.T) { - const n = 25 - const salt = 17 - a := new(Vector).Resize(n, 0) - for i := 0; i < n; i++ { - a.Set(i, int2Value(salt*i)) - } - count := 0 - a.Do(func(e interface{}) { - i := intf2Value(e) - if i != int2Value(count*salt) { - t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(a), "should visit", n, "values; did visit", count) - } - - b := new(Vector).Resize(n, 0) - for i := 0; i < n; i++ { - (*b)[i] = int2Value(salt * i) - } - count = 0 - b.Do(func(e interface{}) { - i := intf2Value(e) - if i != int2Value(count*salt) { - t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(b), "b) should visit", n, "values; did visit", count) - } - - var c Vector - c.Resize(n, 0) - for i := 0; i < n; i++ { - c[i] = int2Value(salt * i) - } - count = 0 - c.Do(func(e interface{}) { - i := intf2Value(e) - if i != int2Value(count*salt) { - t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) - } - count++ - }) - if count != n { - t.Error(tname(c), "c) should visit", n, "values; did visit", count) - } - -} - -func TestVectorCopy(t *testing.T) { - // verify Copy() returns a copy, not simply a slice of the original vector - const Len = 10 - var src Vector - for i := 0; i < Len; i++ { - src.Push(int2Value(i * i)) - } - dest := src.Copy() - for i := 0; i < Len; i++ { - src[i] = int2Value(-1) - v := elem2Value(dest[i]) - if v != int2Value(i*i) { - t.Error(tname(src), "expected", i*i, "got", v) - } - } -} diff --git a/libgo/go/crypto/bcrypt/base64.go b/libgo/go/crypto/bcrypt/base64.go new file mode 100644 index 0000000..ed6cea7 --- /dev/null +++ b/libgo/go/crypto/bcrypt/base64.go @@ -0,0 +1,38 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bcrypt + +import ( + "encoding/base64" + "os" +) + +const alphabet = "./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + +var bcEncoding = base64.NewEncoding(alphabet) + +func base64Encode(src []byte) []byte { + n := bcEncoding.EncodedLen(len(src)) + dst := make([]byte, n) + bcEncoding.Encode(dst, src) + for dst[n-1] == '=' { + n-- + } + return dst[:n] +} + +func base64Decode(src []byte) ([]byte, os.Error) { + numOfEquals := 4 - (len(src) % 4) + for i := 0; i < numOfEquals; i++ { + src = append(src, '=') + } + + dst := make([]byte, bcEncoding.DecodedLen(len(src))) + n, err := bcEncoding.Decode(dst, src) + if err != nil { + return nil, err + } + return dst[:n], nil +} diff --git a/libgo/go/crypto/bcrypt/bcrypt.go b/libgo/go/crypto/bcrypt/bcrypt.go new file mode 100644 index 0000000..1e8ccfa --- /dev/null +++ b/libgo/go/crypto/bcrypt/bcrypt.go @@ -0,0 +1,282 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bcrypt implements Provos and Mazières's bcrypt adapative hashing +// algorithm. See http://www.usenix.org/event/usenix99/provos/provos.pdf +package bcrypt + +// The code is a port of Provos and Mazières's C implementation. +import ( + "crypto/blowfish" + "crypto/rand" + "crypto/subtle" + "fmt" + "io" + "os" + "strconv" +) + +const ( + MinCost int = 4 // the minimum allowable cost as passed in to GenerateFromPassword + MaxCost int = 31 // the maximum allowable cost as passed in to GenerateFromPassword + DefaultCost int = 10 // the cost that will actually be set if a cost below MinCost is passed into GenerateFromPassword +) + +// The error returned from CompareHashAndPassword when a password and hash do +// not match. +var MismatchedHashAndPasswordError = os.NewError("crypto/bcrypt: hashedPassword is not the hash of the given password") + +// The error returned from CompareHashAndPassword when a hash is too short to +// be a bcrypt hash. +var HashTooShortError = os.NewError("crypto/bcrypt: hashedSecret too short to be a bcrypted password") + +// The error returned from CompareHashAndPassword when a hash was created with +// a bcrypt algorithm newer than this implementation. +type HashVersionTooNewError byte + +func (hv HashVersionTooNewError) String() string { + return fmt.Sprintf("crypto/bcrypt: bcrypt algorithm version '%c' requested is newer than current version '%c'", byte(hv), majorVersion) +} + +// The error returned from CompareHashAndPassword when a hash starts with something other than '$' +type InvalidHashPrefixError byte + +func (ih InvalidHashPrefixError) String() string { + return fmt.Sprintf("crypto/bcrypt: bcrypt hashes must start with '$', but hashedSecret started with '%c'", byte(ih)) +} + +type InvalidCostError int + +func (ic InvalidCostError) String() string { + return fmt.Sprintf("crypto/bcrypt: cost %d is outside allowed range (%d,%d)", int(ic), int(MinCost), int(MaxCost)) +} + +const ( + majorVersion = '2' + minorVersion = 'a' + maxSaltSize = 16 + maxCryptedHashSize = 23 + encodedSaltSize = 22 + encodedHashSize = 31 + minHashSize = 59 +) + +// magicCipherData is an IV for the 64 Blowfish encryption calls in +// bcrypt(). It's the string "OrpheanBeholderScryDoubt" in big-endian bytes. +var magicCipherData = []byte{ + 0x4f, 0x72, 0x70, 0x68, + 0x65, 0x61, 0x6e, 0x42, + 0x65, 0x68, 0x6f, 0x6c, + 0x64, 0x65, 0x72, 0x53, + 0x63, 0x72, 0x79, 0x44, + 0x6f, 0x75, 0x62, 0x74, +} + +type hashed struct { + hash []byte + salt []byte + cost uint32 // allowed range is MinCost to MaxCost + major byte + minor byte +} + +// GenerateFromPassword returns the bcrypt hash of the password at the given +// cost. If the cost given is less than MinCost, the cost will be set to +// MinCost, instead. Use CompareHashAndPassword, as defined in this package, +// to compare the returned hashed password with its cleartext version. +func GenerateFromPassword(password []byte, cost int) ([]byte, os.Error) { + p, err := newFromPassword(password, cost) + if err != nil { + return nil, err + } + return p.Hash(), nil +} + +// CompareHashAndPassword compares a bcrypt hashed password with its possible +// plaintext equivalent. Note: Using bytes.Equal for this job is +// insecure. Returns nil on success, or an error on failure. +func CompareHashAndPassword(hashedPassword, password []byte) os.Error { + p, err := newFromHash(hashedPassword) + if err != nil { + return err + } + + otherHash, err := bcrypt(password, p.cost, p.salt) + if err != nil { + return err + } + + otherP := &hashed{otherHash, p.salt, p.cost, p.major, p.minor} + if subtle.ConstantTimeCompare(p.Hash(), otherP.Hash()) == 1 { + return nil + } + + return MismatchedHashAndPasswordError +} + +func newFromPassword(password []byte, cost int) (*hashed, os.Error) { + if cost < MinCost { + cost = DefaultCost + } + p := new(hashed) + p.major = majorVersion + p.minor = minorVersion + + err := checkCost(cost) + if err != nil { + return nil, err + } + p.cost = uint32(cost) + + unencodedSalt := make([]byte, maxSaltSize) + _, err = io.ReadFull(rand.Reader, unencodedSalt) + if err != nil { + return nil, err + } + + p.salt = base64Encode(unencodedSalt) + hash, err := bcrypt(password, p.cost, p.salt) + if err != nil { + return nil, err + } + p.hash = hash + return p, err +} + +func newFromHash(hashedSecret []byte) (*hashed, os.Error) { + if len(hashedSecret) < minHashSize { + return nil, HashTooShortError + } + p := new(hashed) + n, err := p.decodeVersion(hashedSecret) + if err != nil { + return nil, err + } + hashedSecret = hashedSecret[n:] + n, err = p.decodeCost(hashedSecret) + if err != nil { + return nil, err + } + hashedSecret = hashedSecret[n:] + + // The "+2" is here because we'll have to append at most 2 '=' to the salt + // when base64 decoding it in expensiveBlowfishSetup(). + p.salt = make([]byte, encodedSaltSize, encodedSaltSize+2) + copy(p.salt, hashedSecret[:encodedSaltSize]) + + hashedSecret = hashedSecret[encodedSaltSize:] + p.hash = make([]byte, len(hashedSecret)) + copy(p.hash, hashedSecret) + + return p, nil +} + +func bcrypt(password []byte, cost uint32, salt []byte) ([]byte, os.Error) { + cipherData := make([]byte, len(magicCipherData)) + copy(cipherData, magicCipherData) + + c, err := expensiveBlowfishSetup(password, cost, salt) + if err != nil { + return nil, err + } + + for i := 0; i < 24; i += 8 { + for j := 0; j < 64; j++ { + c.Encrypt(cipherData[i:i+8], cipherData[i:i+8]) + } + } + + // Bug compatibility with C bcrypt implementations. We only encode 23 of + // the 24 bytes encrypted. + hsh := base64Encode(cipherData[:maxCryptedHashSize]) + return hsh, nil +} + +func expensiveBlowfishSetup(key []byte, cost uint32, salt []byte) (*blowfish.Cipher, os.Error) { + + csalt, err := base64Decode(salt) + if err != nil { + return nil, err + } + + // Bug compatibility with C bcrypt implementations. They use the trailing + // NULL in the key string during expansion. + ckey := append(key, 0) + + c, err := blowfish.NewSaltedCipher(ckey, csalt) + if err != nil { + return nil, err + } + + rounds := 1 << cost + for i := 0; i < rounds; i++ { + blowfish.ExpandKey(ckey, c) + blowfish.ExpandKey(csalt, c) + } + + return c, nil +} + +func (p *hashed) Hash() []byte { + arr := make([]byte, 60) + arr[0] = '$' + arr[1] = p.major + n := 2 + if p.minor != 0 { + arr[2] = p.minor + n = 3 + } + arr[n] = '$' + n += 1 + copy(arr[n:], []byte(fmt.Sprintf("%02d", p.cost))) + n += 2 + arr[n] = '$' + n += 1 + copy(arr[n:], p.salt) + n += encodedSaltSize + copy(arr[n:], p.hash) + n += encodedHashSize + return arr[:n] +} + +func (p *hashed) decodeVersion(sbytes []byte) (int, os.Error) { + if sbytes[0] != '$' { + return -1, InvalidHashPrefixError(sbytes[0]) + } + if sbytes[1] > majorVersion { + return -1, HashVersionTooNewError(sbytes[1]) + } + p.major = sbytes[1] + n := 3 + if sbytes[2] != '$' { + p.minor = sbytes[2] + n++ + } + return n, nil +} + +// sbytes should begin where decodeVersion left off. +func (p *hashed) decodeCost(sbytes []byte) (int, os.Error) { + cost, err := strconv.Atoi(string(sbytes[0:2])) + if err != nil { + return -1, err + } + err = checkCost(cost) + if err != nil { + return -1, err + } + p.cost = uint32(cost) + return 3, nil +} + +func (p *hashed) String() string { + return fmt.Sprintf("&{hash: %#v, salt: %#v, cost: %d, major: %c, minor: %c}", string(p.hash), p.salt, p.cost, p.major, p.minor) +} + +func checkCost(cost int) os.Error { + if cost < MinCost || cost > MaxCost { + return InvalidCostError(cost) + } + return nil +} diff --git a/libgo/go/crypto/bcrypt/bcrypt_test.go b/libgo/go/crypto/bcrypt/bcrypt_test.go new file mode 100644 index 0000000..89eca0a --- /dev/null +++ b/libgo/go/crypto/bcrypt/bcrypt_test.go @@ -0,0 +1,195 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bcrypt + +import ( + "bytes" + "os" + "testing" +) + +func TestBcryptingIsEasy(t *testing.T) { + pass := []byte("mypassword") + hp, err := GenerateFromPassword(pass, 0) + if err != nil { + t.Fatalf("GenerateFromPassword error: %s", err) + } + + if CompareHashAndPassword(hp, pass) != nil { + t.Errorf("%v should hash %s correctly", hp, pass) + } + + notPass := "notthepass" + err = CompareHashAndPassword(hp, []byte(notPass)) + if err != MismatchedHashAndPasswordError { + t.Errorf("%v and %s should be mismatched", hp, notPass) + } +} + +func TestBcryptingIsCorrect(t *testing.T) { + pass := []byte("allmine") + salt := []byte("XajjQvNhvvRt5GSeFk1xFe") + expectedHash := []byte("$2a$10$XajjQvNhvvRt5GSeFk1xFeyqRrsxkhBkUiQeg0dt.wU1qD4aFDcga") + + hash, err := bcrypt(pass, 10, salt) + if err != nil { + t.Fatalf("bcrypt blew up: %v", err) + } + if !bytes.HasSuffix(expectedHash, hash) { + t.Errorf("%v should be the suffix of %v", hash, expectedHash) + } + + h, err := newFromHash(expectedHash) + if err != nil { + t.Errorf("Unable to parse %s: %v", string(expectedHash), err) + } + + // This is not the safe way to compare these hashes. We do this only for + // testing clarity. Use bcrypt.CompareHashAndPassword() + if err == nil && !bytes.Equal(expectedHash, h.Hash()) { + t.Errorf("Parsed hash %v should equal %v", h.Hash(), expectedHash) + } +} + +func TestTooLongPasswordsWork(t *testing.T) { + salt := []byte("XajjQvNhvvRt5GSeFk1xFe") + // One byte over the usual 56 byte limit that blowfish has + tooLongPass := []byte("012345678901234567890123456789012345678901234567890123456") + tooLongExpected := []byte("$2a$10$XajjQvNhvvRt5GSeFk1xFe5l47dONXg781AmZtd869sO8zfsHuw7C") + hash, err := bcrypt(tooLongPass, 10, salt) + if err != nil { + t.Fatalf("bcrypt blew up on long password: %v", err) + } + if !bytes.HasSuffix(tooLongExpected, hash) { + t.Errorf("%v should be the suffix of %v", hash, tooLongExpected) + } +} + +type InvalidHashTest struct { + err os.Error + hash []byte +} + +var invalidTests = []InvalidHashTest{ + {HashTooShortError, []byte("$2a$10$fooo")}, + {HashTooShortError, []byte("$2a")}, + {HashVersionTooNewError('3'), []byte("$3a$10$sssssssssssssssssssssshhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh")}, + {InvalidHashPrefixError('%'), []byte("%2a$10$sssssssssssssssssssssshhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh")}, + {InvalidCostError(32), []byte("$2a$32$sssssssssssssssssssssshhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh")}, +} + +func TestInvalidHashErrors(t *testing.T) { + check := func(name string, expected, err os.Error) { + if err == nil { + t.Errorf("%s: Should have returned an error", name) + } + if err != nil && err != expected { + t.Errorf("%s gave err %v but should have given %v", name, err.String(), expected.String()) + } + } + for _, iht := range invalidTests { + _, err := newFromHash(iht.hash) + check("newFromHash", iht.err, err) + err = CompareHashAndPassword(iht.hash, []byte("anything")) + check("CompareHashAndPassword", iht.err, err) + } +} + +func TestUnpaddedBase64Encoding(t *testing.T) { + original := []byte{101, 201, 101, 75, 19, 227, 199, 20, 239, 236, 133, 32, 30, 109, 243, 30} + encodedOriginal := []byte("XajjQvNhvvRt5GSeFk1xFe") + + encoded := base64Encode(original) + + if !bytes.Equal(encodedOriginal, encoded) { + t.Errorf("Encoded %v should have equaled %v", encoded, encodedOriginal) + } + + decoded, err := base64Decode(encodedOriginal) + if err != nil { + t.Fatalf("base64Decode blew up: %s", err) + } + + if !bytes.Equal(decoded, original) { + t.Errorf("Decoded %v should have equaled %v", decoded, original) + } +} + +func TestCost(t *testing.T) { + if testing.Short() { + return + } + + pass := []byte("mypassword") + + for c := 0; c < MinCost; c++ { + p, _ := newFromPassword(pass, c) + if p.cost != uint32(DefaultCost) { + t.Errorf("newFromPassword should default costs below %d to %d, but was %d", MinCost, DefaultCost, p.cost) + } + } + + p, _ := newFromPassword(pass, 14) + if p.cost != 14 { + t.Errorf("newFromPassword should default cost to 14, but was %d", p.cost) + } + + hp, _ := newFromHash(p.Hash()) + if p.cost != hp.cost { + t.Errorf("newFromHash should maintain the cost at %d, but was %d", p.cost, hp.cost) + } + + _, err := newFromPassword(pass, 32) + if err == nil { + t.Fatalf("newFromPassword: should return a cost error") + } + if err != InvalidCostError(32) { + t.Errorf("newFromPassword: should return cost error, got %#v", err) + } +} + +func TestCostReturnsWithLeadingZeroes(t *testing.T) { + hp, _ := newFromPassword([]byte("abcdefgh"), 7) + cost := hp.Hash()[4:7] + expected := []byte("07$") + + if !bytes.Equal(expected, cost) { + t.Errorf("single digit costs in hash should have leading zeros: was %v instead of %v", cost, expected) + } +} + +func TestMinorNotRequired(t *testing.T) { + noMinorHash := []byte("$2$10$XajjQvNhvvRt5GSeFk1xFeyqRrsxkhBkUiQeg0dt.wU1qD4aFDcga") + h, err := newFromHash(noMinorHash) + if err != nil { + t.Fatalf("No minor hash blew up: %s", err) + } + if h.minor != 0 { + t.Errorf("Should leave minor version at 0, but was %d", h.minor) + } + + if !bytes.Equal(noMinorHash, h.Hash()) { + t.Errorf("Should generate hash %v, but created %v", noMinorHash, h.Hash()) + } +} + +func BenchmarkEqual(b *testing.B) { + b.StopTimer() + passwd := []byte("somepasswordyoulike") + hash, _ := GenerateFromPassword(passwd, 10) + b.StartTimer() + for i := 0; i < b.N; i++ { + CompareHashAndPassword(hash, passwd) + } +} + +func BenchmarkGeneration(b *testing.B) { + b.StopTimer() + passwd := []byte("mylongpassword1234") + b.StartTimer() + for i := 0; i < b.N; i++ { + GenerateFromPassword(passwd, 10) + } +} diff --git a/libgo/go/crypto/blowfish/block.go b/libgo/go/crypto/blowfish/block.go index 7fbe7ee..326292d 100644 --- a/libgo/go/crypto/blowfish/block.go +++ b/libgo/go/crypto/blowfish/block.go @@ -4,13 +4,12 @@ package blowfish -func expandKey(key []byte, c *Cipher) { - copy(c.p[0:], p[0:]) - copy(c.s0[0:], s0[0:]) - copy(c.s1[0:], s1[0:]) - copy(c.s2[0:], s2[0:]) - copy(c.s3[0:], s3[0:]) - +// ExpandKey performs a key expansion on the given *Cipher. Specifically, it +// performs the Blowfish algorithm's key schedule which sets up the *Cipher's +// pi and substitution tables for calls to Encrypt. This is used, primarily, +// by the bcrypt package to reuse the Blowfish key schedule during its +// set up. It's unlikely that you need to use this directly. +func ExpandKey(key []byte, c *Cipher) { j := 0 for i := 0; i < 18; i++ { var d uint32 @@ -48,6 +47,98 @@ func expandKey(key []byte, c *Cipher) { } } +// This is similar to ExpandKey, but folds the salt during the key +// schedule. While ExpandKey is essentially expandKeyWithSalt with an all-zero +// salt passed in, reusing ExpandKey turns out to be a place of inefficiency +// and specializing it here is useful. +func expandKeyWithSalt(key []byte, salt []byte, c *Cipher) { + j := 0 + expandedKey := make([]uint32, 18) + for i := 0; i < 18; i++ { + var d uint32 + for k := 0; k < 4; k++ { + d = d<<8 | uint32(key[j])&0x000000FF + j++ + if j >= len(key) { + j = 0 + } + } + expandedKey[i] = d + c.p[i] ^= d + } + + j = 0 + expandedSalt := make([]uint32, 18) + for i := 0; i < 18; i++ { + var d uint32 + for k := 0; k < 4; k++ { + d = d<<8 | uint32(salt[j])&0x000000FF + j++ + if j >= len(salt) { + j = 0 + } + } + expandedSalt[i] = d + } + + var l, r uint32 + for i := 0; i < 18; i += 2 { + l ^= expandedSalt[i&2] + r ^= expandedSalt[(i&2)+1] + l, r = encryptBlock(l, r, c) + c.p[i], c.p[i+1] = l, r + } + + for i := 0; i < 256; i += 4 { + l ^= expandedSalt[2] + r ^= expandedSalt[3] + l, r = encryptBlock(l, r, c) + c.s0[i], c.s0[i+1] = l, r + + l ^= expandedSalt[0] + r ^= expandedSalt[1] + l, r = encryptBlock(l, r, c) + c.s0[i+2], c.s0[i+3] = l, r + + } + + for i := 0; i < 256; i += 4 { + l ^= expandedSalt[2] + r ^= expandedSalt[3] + l, r = encryptBlock(l, r, c) + c.s1[i], c.s1[i+1] = l, r + + l ^= expandedSalt[0] + r ^= expandedSalt[1] + l, r = encryptBlock(l, r, c) + c.s1[i+2], c.s1[i+3] = l, r + } + + for i := 0; i < 256; i += 4 { + l ^= expandedSalt[2] + r ^= expandedSalt[3] + l, r = encryptBlock(l, r, c) + c.s2[i], c.s2[i+1] = l, r + + l ^= expandedSalt[0] + r ^= expandedSalt[1] + l, r = encryptBlock(l, r, c) + c.s2[i+2], c.s2[i+3] = l, r + } + + for i := 0; i < 256; i += 4 { + l ^= expandedSalt[2] + r ^= expandedSalt[3] + l, r = encryptBlock(l, r, c) + c.s3[i], c.s3[i+1] = l, r + + l ^= expandedSalt[0] + r ^= expandedSalt[1] + l, r = encryptBlock(l, r, c) + c.s3[i+2], c.s3[i+3] = l, r + } +} + func encryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { xl, xr := l, r xl ^= c.p[0] diff --git a/libgo/go/crypto/blowfish/blowfish_test.go b/libgo/go/crypto/blowfish/blowfish_test.go index 3a7ab6c..1038d2e 100644 --- a/libgo/go/crypto/blowfish/blowfish_test.go +++ b/libgo/go/crypto/blowfish/blowfish_test.go @@ -190,3 +190,21 @@ func TestCipherDecrypt(t *testing.T) { } } } + +func TestSaltedCipherKeyLength(t *testing.T) { + var key []byte + for i := 0; i < 4; i++ { + _, err := NewSaltedCipher(key, []byte{'a'}) + if err != KeySizeError(i) { + t.Errorf("NewSaltedCipher with short key, gave error %#v, expected %#v", err, KeySizeError(i)) + } + key = append(key, 'a') + } + + // A 57-byte key. One over the typical blowfish restriction. + key = []byte("012345678901234567890123456789012345678901234567890123456") + _, err := NewSaltedCipher(key, []byte{'a'}) + if err != nil { + t.Errorf("NewSaltedCipher with long key, gave error %#v", err) + } +} diff --git a/libgo/go/crypto/blowfish/cipher.go b/libgo/go/crypto/blowfish/cipher.go index 6c37dfe..3439825 100644 --- a/libgo/go/crypto/blowfish/cipher.go +++ b/libgo/go/crypto/blowfish/cipher.go @@ -31,12 +31,28 @@ func (k KeySizeError) String() string { // NewCipher creates and returns a Cipher. // The key argument should be the Blowfish key, 4 to 56 bytes. func NewCipher(key []byte) (*Cipher, os.Error) { + var result Cipher k := len(key) if k < 4 || k > 56 { return nil, KeySizeError(k) } + initCipher(key, &result) + ExpandKey(key, &result) + return &result, nil +} + +// NewSaltedCipher creates a returns a Cipher that folds a salt into its key +// schedule. For most purposes, NewCipher, instead of NewSaltedCipher, is +// sufficient and desirable. For bcrypt compatiblity, the key can be over 56 +// bytes. +func NewSaltedCipher(key, salt []byte) (*Cipher, os.Error) { var result Cipher - expandKey(key, &result) + k := len(key) + if k < 4 { + return nil, KeySizeError(k) + } + initCipher(key, &result) + expandKeyWithSalt(key, salt, &result) return &result, nil } @@ -77,3 +93,11 @@ func (c *Cipher) Reset() { zero(c.s2[0:]) zero(c.s3[0:]) } + +func initCipher(key []byte, c *Cipher) { + copy(c.p[0:], p[0:]) + copy(c.s0[0:], s0[0:]) + copy(c.s1[0:], s1[0:]) + copy(c.s2[0:], s2[0:]) + copy(c.s3[0:], s3[0:]) +} diff --git a/libgo/go/crypto/elliptic/elliptic_test.go b/libgo/go/crypto/elliptic/elliptic_test.go index b7e7f03..9578364 100644 --- a/libgo/go/crypto/elliptic/elliptic_test.go +++ b/libgo/go/crypto/elliptic/elliptic_test.go @@ -295,7 +295,7 @@ func TestBaseMult(t *testing.T) { } x, y := p224.ScalarBaseMult(k.Bytes()) if fmt.Sprintf("%x", x) != e.x || fmt.Sprintf("%x", y) != e.y { - t.Errorf("%d: bad output for k=%s: got (%x, %s), want (%s, %s)", i, e.k, x, y, e.x, e.y) + t.Errorf("%d: bad output for k=%s: got (%x, %s), want (%x, %s)", i, e.k, x, y, e.x, e.y) } if testing.Short() && i > 5 { break diff --git a/libgo/go/crypto/ocsp/ocsp_test.go b/libgo/go/crypto/ocsp/ocsp_test.go index f988979..7be3721 100644 --- a/libgo/go/crypto/ocsp/ocsp_test.go +++ b/libgo/go/crypto/ocsp/ocsp_test.go @@ -15,7 +15,7 @@ func TestOCSPDecode(t *testing.T) { t.Error(err) } - expected := Response{Status: 0, SerialNumber: []byte{0x1, 0xd0, 0xfa}, RevocationReason: 0, ThisUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 15, Minute: 1, Second: 5, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NextUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 18, Minute: 35, Second: 17, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}} + expected := Response{Status: 0, SerialNumber: []byte{0x1, 0xd0, 0xfa}, RevocationReason: 0, ThisUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 15, Minute: 1, Second: 5, ZoneOffset: 0, Zone: "UTC"}, NextUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 18, Minute: 35, Second: 17, ZoneOffset: 0, Zone: "UTC"}} if !reflect.DeepEqual(resp.ThisUpdate, resp.ThisUpdate) { t.Errorf("resp.ThisUpdate: got %d, want %d", resp.ThisUpdate, expected.ThisUpdate) diff --git a/libgo/go/crypto/rand/rand_unix.go b/libgo/go/crypto/rand/rand_unix.go index 3a06aa8..76a7365 100644 --- a/libgo/go/crypto/rand/rand_unix.go +++ b/libgo/go/crypto/rand/rand_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + // Unix cryptographically secure pseudorandom number // generator. diff --git a/libgo/go/crypto/tls/cipher_suites.go b/libgo/go/crypto/tls/cipher_suites.go index bc7b0d3..0c62251 100644 --- a/libgo/go/crypto/tls/cipher_suites.go +++ b/libgo/go/crypto/tls/cipher_suites.go @@ -7,8 +7,10 @@ package tls import ( "crypto/aes" "crypto/cipher" + "crypto/des" "crypto/hmac" "crypto/rc4" + "crypto/sha1" "crypto/x509" "hash" "os" @@ -23,7 +25,7 @@ type keyAgreement interface { // ServerKeyExchange message, generateServerKeyExchange can return nil, // nil. generateServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) - processClientKeyExchange(*Config, *clientKeyExchangeMsg) ([]byte, os.Error) + processClientKeyExchange(*Config, *clientKeyExchangeMsg, uint16) ([]byte, os.Error) // On the client side, the next two methods are called in order. @@ -46,14 +48,16 @@ type cipherSuite struct { // and point format that we can handle. elliptic bool cipher func(key, iv []byte, isRead bool) interface{} - mac func(macKey []byte) hash.Hash + mac func(version uint16, macKey []byte) macFunction } var cipherSuites = map[uint16]*cipherSuite{ - TLS_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, rsaKA, false, cipherRC4, hmacSHA1}, - TLS_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, rsaKA, false, cipherAES, hmacSHA1}, - TLS_ECDHE_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, ecdheRSAKA, true, cipherRC4, hmacSHA1}, - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, ecdheRSAKA, true, cipherAES, hmacSHA1}, + TLS_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, rsaKA, false, cipherRC4, macSHA1}, + TLS_RSA_WITH_3DES_EDE_CBC_SHA: &cipherSuite{24, 20, 8, rsaKA, false, cipher3DES, macSHA1}, + TLS_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, rsaKA, false, cipherAES, macSHA1}, + TLS_ECDHE_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, ecdheRSAKA, true, cipherRC4, macSHA1}, + TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA: &cipherSuite{24, 20, 8, ecdheRSAKA, true, cipher3DES, macSHA1}, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, ecdheRSAKA, true, cipherAES, macSHA1}, } func cipherRC4(key, iv []byte, isRead bool) interface{} { @@ -61,6 +65,14 @@ func cipherRC4(key, iv []byte, isRead bool) interface{} { return cipher } +func cipher3DES(key, iv []byte, isRead bool) interface{} { + block, _ := des.NewTripleDESCipher(key) + if isRead { + return cipher.NewCBCDecrypter(block, iv) + } + return cipher.NewCBCEncrypter(block, iv) +} + func cipherAES(key, iv []byte, isRead bool) interface{} { block, _ := aes.NewCipher(key) if isRead { @@ -69,8 +81,75 @@ func cipherAES(key, iv []byte, isRead bool) interface{} { return cipher.NewCBCEncrypter(block, iv) } -func hmacSHA1(key []byte) hash.Hash { - return hmac.NewSHA1(key) +// macSHA1 returns a macFunction for the given protocol version. +func macSHA1(version uint16, key []byte) macFunction { + if version == versionSSL30 { + mac := ssl30MAC{ + h: sha1.New(), + key: make([]byte, len(key)), + } + copy(mac.key, key) + return mac + } + return tls10MAC{hmac.NewSHA1(key)} +} + +type macFunction interface { + Size() int + MAC(seq, data []byte) []byte +} + +// ssl30MAC implements the SSLv3 MAC function, as defined in +// www.mozilla.org/projects/security/pki/nss/ssl/draft302.txt section 5.2.3.1 +type ssl30MAC struct { + h hash.Hash + key []byte +} + +func (s ssl30MAC) Size() int { + return s.h.Size() +} + +var ssl30Pad1 = [48]byte{0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36} + +var ssl30Pad2 = [48]byte{0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c} + +func (s ssl30MAC) MAC(seq, record []byte) []byte { + padLength := 48 + if s.h.Size() == 20 { + padLength = 40 + } + + s.h.Reset() + s.h.Write(s.key) + s.h.Write(ssl30Pad1[:padLength]) + s.h.Write(seq) + s.h.Write(record[:1]) + s.h.Write(record[3:5]) + s.h.Write(record[recordHeaderLen:]) + digest := s.h.Sum() + + s.h.Reset() + s.h.Write(s.key) + s.h.Write(ssl30Pad2[:padLength]) + s.h.Write(digest) + return s.h.Sum() +} + +// tls10MAC implements the TLS 1.0 MAC function. RFC 2246, section 6.2.3. +type tls10MAC struct { + h hash.Hash +} + +func (s tls10MAC) Size() int { + return s.h.Size() +} + +func (s tls10MAC) MAC(seq, record []byte) []byte { + s.h.Reset() + s.h.Write(seq) + s.h.Write(record) + return s.h.Sum() } func rsaKA() keyAgreement { @@ -95,8 +174,10 @@ func mutualCipherSuite(have []uint16, want uint16) (suite *cipherSuite, id uint1 // A list of the possible cipher suite ids. Taken from // http://www.iana.org/assignments/tls-parameters/tls-parameters.xml const ( - TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 - TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002f - TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xc011 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xc013 + TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + TLS_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x000a + TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002f + TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xc011 + TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xc012 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xc013 ) diff --git a/libgo/go/crypto/tls/common.go b/libgo/go/crypto/tls/common.go index 3efac9c..ea52085 100644 --- a/libgo/go/crypto/tls/common.go +++ b/libgo/go/crypto/tls/common.go @@ -9,7 +9,7 @@ import ( "crypto/rsa" "crypto/x509" "io" - "io/ioutil" + "strings" "sync" "time" ) @@ -20,8 +20,11 @@ const ( recordHeaderLen = 5 // record header length maxHandshake = 65536 // maximum handshake we support (protocol max is 16 MB) - minVersion = 0x0301 // minimum supported version - TLS 1.0 - maxVersion = 0x0301 // maximum supported version - TLS 1.0 + versionSSL30 = 0x0300 + versionTLS10 = 0x0301 + + minVersion = versionSSL30 + maxVersion = versionTLS10 ) // TLS record types. @@ -98,6 +101,10 @@ type ConnectionState struct { NegotiatedProtocol string NegotiatedProtocolIsMutual bool + // ServerName contains the server name indicated by the client, if any. + // (Only valid for server connections.) + ServerName string + // the certificate chain that was presented by the other side PeerCertificates []*x509.Certificate // the verified certificate chains built from PeerCertificates. @@ -121,6 +128,14 @@ type Config struct { // Server configurations must include at least one certificate. Certificates []Certificate + // NameToCertificate maps from a certificate name to an element of + // Certificates. Note that a certificate name can be of the form + // '*.example.com' and so doesn't have to be a domain name as such. + // See Config.BuildNameToCertificate + // The nil value causes the first element of Certificates to be used + // for all connections. + NameToCertificate map[string]*Certificate + // RootCAs defines the set of root certificate authorities // that clients use when verifying server certificates. // If RootCAs is nil, TLS uses the host's root CA set. @@ -139,6 +154,14 @@ type Config struct { // anything more than self-signed. AuthenticateClient bool + // InsecureSkipVerify controls whether a client verifies the + // server's certificate chain and host name. + // If InsecureSkipVerify is true, TLS accepts any certificate + // presented by the server and any host name in that certificate. + // In this mode, TLS is susceptible to man-in-the-middle attacks. + // This should be used only for testing. + InsecureSkipVerify bool + // CipherSuites is a list of supported cipher suites. If CipherSuites // is nil, TLS uses a list of suites supported by the implementation. CipherSuites []uint16 @@ -176,6 +199,59 @@ func (c *Config) cipherSuites() []uint16 { return s } +// getCertificateForName returns the best certificate for the given name, +// defaulting to the first element of c.Certificates if there are no good +// options. +func (c *Config) getCertificateForName(name string) *Certificate { + if len(c.Certificates) == 1 || c.NameToCertificate == nil { + // There's only one choice, so no point doing any work. + return &c.Certificates[0] + } + + name = strings.ToLower(name) + for len(name) > 0 && name[len(name)-1] == '.' { + name = name[:len(name)-1] + } + + if cert, ok := c.NameToCertificate[name]; ok { + return cert + } + + // try replacing labels in the name with wildcards until we get a + // match. + labels := strings.Split(name, ".") + for i := range labels { + labels[i] = "*" + candidate := strings.Join(labels, ".") + if cert, ok := c.NameToCertificate[candidate]; ok { + return cert + } + } + + // If nothing matches, return the first certificate. + return &c.Certificates[0] +} + +// BuildNameToCertificate parses c.Certificates and builds c.NameToCertificate +// from the CommonName and SubjectAlternateName fields of each of the leaf +// certificates. +func (c *Config) BuildNameToCertificate() { + c.NameToCertificate = make(map[string]*Certificate) + for i := range c.Certificates { + cert := &c.Certificates[i] + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + continue + } + if len(x509Cert.Subject.CommonName) > 0 { + c.NameToCertificate[x509Cert.Subject.CommonName] = cert + } + for _, san := range x509Cert.DNSNames { + c.NameToCertificate[san] = cert + } + } +} + // A Certificate is a chain of one or more certificates, leaf first. type Certificate struct { Certificate [][]byte @@ -215,15 +291,6 @@ func defaultConfig() *Config { return &emptyConfig } -// Possible certificate files; stop after finding one. -// On OS X we should really be using the Directory Services keychain -// but that requires a lot of Mach goo to get at. Instead we use -// the same root set that curl uses. -var certFiles = []string{ - "/etc/ssl/certs/ca-certificates.crt", // Linux etc - "/usr/share/curl/curl-ca-bundle.crt", // OS X -} - var once sync.Once func defaultRoots() *x509.CertPool { @@ -241,21 +308,10 @@ func initDefaults() { initDefaultCipherSuites() } -var varDefaultRoots *x509.CertPool - -func initDefaultRoots() { - roots := x509.NewCertPool() - for _, file := range certFiles { - data, err := ioutil.ReadFile(file) - if err == nil { - roots.AppendCertsFromPEM(data) - break - } - } - varDefaultRoots = roots -} - -var varDefaultCipherSuites []uint16 +var ( + varDefaultRoots *x509.CertPool + varDefaultCipherSuites []uint16 +) func initDefaultCipherSuites() { varDefaultCipherSuites = make([]uint16, len(cipherSuites)) diff --git a/libgo/go/crypto/tls/conn.go b/libgo/go/crypto/tls/conn.go index fac65af..9bca7d9 100644 --- a/libgo/go/crypto/tls/conn.go +++ b/libgo/go/crypto/tls/conn.go @@ -11,7 +11,6 @@ import ( "crypto/cipher" "crypto/subtle" "crypto/x509" - "hash" "io" "net" "os" @@ -37,6 +36,8 @@ type Conn struct { // verifiedChains contains the certificate chains that we built, as // opposed to the ones presented by the server. verifiedChains [][]*x509.Certificate + // serverName contains the server name indicated by the client, if any. + serverName string clientProtocol string clientProtocolFallback bool @@ -108,18 +109,20 @@ func (c *Conn) SetWriteTimeout(nsec int64) os.Error { // connection, either sending or receiving. type halfConn struct { sync.Mutex - cipher interface{} // cipher algorithm - mac hash.Hash // MAC algorithm - seq [8]byte // 64-bit sequence number - bfree *block // list of free blocks + version uint16 // protocol version + cipher interface{} // cipher algorithm + mac macFunction + seq [8]byte // 64-bit sequence number + bfree *block // list of free blocks nextCipher interface{} // next encryption state - nextMac hash.Hash // next MAC algorithm + nextMac macFunction // next MAC algorithm } // prepareCipherSpec sets the encryption and MAC states // that a subsequent changeCipherSpec will use. -func (hc *halfConn) prepareCipherSpec(cipher interface{}, mac hash.Hash) { +func (hc *halfConn) prepareCipherSpec(version uint16, cipher interface{}, mac macFunction) { + hc.version = version hc.nextCipher = cipher hc.nextMac = mac } @@ -197,6 +200,22 @@ func removePadding(payload []byte) ([]byte, byte) { return payload[:len(payload)-int(toRemove)], good } +// removePaddingSSL30 is a replacement for removePadding in the case that the +// protocol version is SSLv3. In this version, the contents of the padding +// are random and cannot be checked. +func removePaddingSSL30(payload []byte) ([]byte, byte) { + if len(payload) < 1 { + return payload, 0 + } + + paddingLen := int(payload[len(payload)-1]) + 1 + if paddingLen > len(payload) { + return payload, 0 + } + + return payload[:len(payload)-paddingLen], 255 +} + func roundUp(a, b int) int { return a + (b-a%b)%b } @@ -226,7 +245,11 @@ func (hc *halfConn) decrypt(b *block) (bool, alert) { } c.CryptBlocks(payload, payload) - payload, paddingGood = removePadding(payload) + if hc.version == versionSSL30 { + payload, paddingGood = removePaddingSSL30(payload) + } else { + payload, paddingGood = removePadding(payload) + } b.resize(recordHeaderLen + len(payload)) // note that we still have a timing side-channel in the @@ -256,13 +279,10 @@ func (hc *halfConn) decrypt(b *block) (bool, alert) { b.data[4] = byte(n) b.resize(recordHeaderLen + n) remoteMAC := payload[n:] - - hc.mac.Reset() - hc.mac.Write(hc.seq[0:]) + localMAC := hc.mac.MAC(hc.seq[0:], b.data) hc.incSeq() - hc.mac.Write(b.data) - if subtle.ConstantTimeCompare(hc.mac.Sum(), remoteMAC) != 1 || paddingGood != 255 { + if subtle.ConstantTimeCompare(localMAC, remoteMAC) != 1 || paddingGood != 255 { return false, alertBadRecordMAC } } @@ -291,11 +311,9 @@ func padToBlockSize(payload []byte, blockSize int) (prefix, finalBlock []byte) { func (hc *halfConn) encrypt(b *block) (bool, alert) { // mac if hc.mac != nil { - hc.mac.Reset() - hc.mac.Write(hc.seq[0:]) + mac := hc.mac.MAC(hc.seq[0:], b.data) hc.incSeq() - hc.mac.Write(b.data) - mac := hc.mac.Sum() + n := len(b.data) b.resize(n + len(mac)) copy(b.data[n:], mac) @@ -470,6 +488,19 @@ Again: if n > maxCiphertext { return c.sendAlert(alertRecordOverflow) } + if !c.haveVers { + // First message, be extra suspicious: + // this might not be a TLS client. + // Bail out before reading a full 'body', if possible. + // The current max version is 3.1. + // If the version is >= 16.0, it's probably not real. + // Similarly, a clientHello message encodes in + // well under a kilobyte. If the length is >= 12 kB, + // it's probably not real. + if (typ != recordTypeAlert && typ != want) || vers >= 0x1000 || n >= 0x3000 { + return c.sendAlert(alertUnexpectedMessage) + } + } if err := b.readFromUntil(c.conn, recordHeaderLen+n); err != nil { if err == os.EOF { err = io.ErrUnexpectedEOF @@ -627,7 +658,9 @@ func (c *Conn) readHandshake() (interface{}, os.Error) { if c.err != nil { return nil, c.err } - c.readRecord(recordTypeHandshake) + if err := c.readRecord(recordTypeHandshake); err != nil { + return nil, err + } } data := c.hand.Bytes() @@ -640,7 +673,9 @@ func (c *Conn) readHandshake() (interface{}, os.Error) { if c.err != nil { return nil, c.err } - c.readRecord(recordTypeHandshake) + if err := c.readRecord(recordTypeHandshake); err != nil { + return nil, err + } } data = c.hand.Next(4 + n) var m handshakeMessage @@ -731,10 +766,18 @@ func (c *Conn) Read(b []byte) (n int, err os.Error) { // Close closes the connection. func (c *Conn) Close() os.Error { - if err := c.Handshake(); err != nil { + var alertErr os.Error + + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if c.handshakeComplete { + alertErr = c.sendAlert(alertCloseNotify) + } + + if err := c.conn.Close(); err != nil { return err } - return c.sendAlert(alertCloseNotify) + return alertErr } // Handshake runs the client or server handshake @@ -769,6 +812,7 @@ func (c *Conn) ConnectionState() ConnectionState { state.CipherSuite = c.cipherSuite state.PeerCertificates = c.peerCertificates state.VerifiedChains = c.verifiedChains + state.ServerName = c.serverName } return state diff --git a/libgo/go/crypto/tls/conn_test.go b/libgo/go/crypto/tls/conn_test.go index f44a50b..5c55514 100644 --- a/libgo/go/crypto/tls/conn_test.go +++ b/libgo/go/crypto/tls/conn_test.go @@ -50,3 +50,57 @@ func TestRemovePadding(t *testing.T) { } } } + +var certExampleCom = `308201403081eda003020102020101300b06092a864886f70d010105301e311c301a060355040a131354657374696e67204365727469666963617465301e170d3131313030313138353835325a170d3132303933303138353835325a301e311c301a060355040a131354657374696e67204365727469666963617465305a300b06092a864886f70d010101034b003048024100bced6e32368599eeddf18796bfd03958a154f87e5b084f96e85136a56b886733592f493f0fc68b0d6b3551781cb95e13c5de458b28d6fb60d20a9129313261410203010001a31a301830160603551d11040f300d820b6578616d706c652e636f6d300b06092a864886f70d0101050341001a0b419d2c74474c6450654e5f10b32bf426ffdf55cad1c52602e7a9151513a3424c70f5960dcd682db0c33769cc1daa3fcdd3db10809d2392ed4a1bf50ced18` + +var certWildcardExampleCom = `308201423081efa003020102020101300b06092a864886f70d010105301e311c301a060355040a131354657374696e67204365727469666963617465301e170d3131313030313139303034365a170d3132303933303139303034365a301e311c301a060355040a131354657374696e67204365727469666963617465305a300b06092a864886f70d010101034b003048024100bced6e32368599eeddf18796bfd03958a154f87e5b084f96e85136a56b886733592f493f0fc68b0d6b3551781cb95e13c5de458b28d6fb60d20a9129313261410203010001a31c301a30180603551d110411300f820d2a2e6578616d706c652e636f6d300b06092a864886f70d0101050341001676f0c9e7c33c1b656ed5a6476c4e2ee9ec8e62df7407accb1875272b2edd0a22096cb2c22598d11604104d604f810eb4b5987ca6bb319c7e6ce48725c54059` + +var certFooExampleCom = `308201443081f1a003020102020101300b06092a864886f70d010105301e311c301a060355040a131354657374696e67204365727469666963617465301e170d3131313030313139303131345a170d3132303933303139303131345a301e311c301a060355040a131354657374696e67204365727469666963617465305a300b06092a864886f70d010101034b003048024100bced6e32368599eeddf18796bfd03958a154f87e5b084f96e85136a56b886733592f493f0fc68b0d6b3551781cb95e13c5de458b28d6fb60d20a9129313261410203010001a31e301c301a0603551d1104133011820f666f6f2e6578616d706c652e636f6d300b06092a864886f70d010105034100646a2a51f2aa2477add854b462cf5207ba16d3213ffb5d3d0eed473fbf09935019192d1d5b8ca6a2407b424cf04d97c4cd9197c83ecf81f0eab9464a1109d09f` + +var certDoubleWildcardExampleCom = `308201443081f1a003020102020101300b06092a864886f70d010105301e311c301a060355040a131354657374696e67204365727469666963617465301e170d3131313030313139303134315a170d3132303933303139303134315a301e311c301a060355040a131354657374696e67204365727469666963617465305a300b06092a864886f70d010101034b003048024100bced6e32368599eeddf18796bfd03958a154f87e5b084f96e85136a56b886733592f493f0fc68b0d6b3551781cb95e13c5de458b28d6fb60d20a9129313261410203010001a31e301c301a0603551d1104133011820f2a2e2a2e6578616d706c652e636f6d300b06092a864886f70d0101050341001c3de267975f56ef57771c6218ef95ecc65102e57bd1defe6f7efea90d9b26cf40de5bd7ad75e46201c7f2a92aaa3e907451e9409f65e28ddb6db80d726290f6` + +func TestCertificateSelection(t *testing.T) { + config := Config{ + Certificates: []Certificate{ + { + Certificate: [][]byte{fromHex(certExampleCom)}, + }, + { + Certificate: [][]byte{fromHex(certWildcardExampleCom)}, + }, + { + Certificate: [][]byte{fromHex(certFooExampleCom)}, + }, + { + Certificate: [][]byte{fromHex(certDoubleWildcardExampleCom)}, + }, + }, + } + + config.BuildNameToCertificate() + + pointerToIndex := func(c *Certificate) int { + for i := range config.Certificates { + if c == &config.Certificates[i] { + return i + } + } + return -1 + } + + if n := pointerToIndex(config.getCertificateForName("example.com")); n != 0 { + t.Errorf("example.com returned certificate %d, not 0", n) + } + if n := pointerToIndex(config.getCertificateForName("bar.example.com")); n != 1 { + t.Errorf("bar.example.com returned certificate %d, not 1", n) + } + if n := pointerToIndex(config.getCertificateForName("foo.example.com")); n != 2 { + t.Errorf("foo.example.com returned certificate %d, not 2", n) + } + if n := pointerToIndex(config.getCertificateForName("foo.bar.example.com")); n != 3 { + t.Errorf("foo.bar.example.com returned certificate %d, not 3", n) + } + if n := pointerToIndex(config.getCertificateForName("foo.bar.baz.example.com")); n != 0 { + t.Errorf("foo.bar.baz.example.com returned certificate %d, not 0", n) + } +} diff --git a/libgo/go/crypto/tls/handshake_client.go b/libgo/go/crypto/tls/handshake_client.go index 15604ce..575a121 100644 --- a/libgo/go/crypto/tls/handshake_client.go +++ b/libgo/go/crypto/tls/handshake_client.go @@ -14,7 +14,7 @@ import ( ) func (c *Conn) clientHandshake() os.Error { - finishedHash := newFinishedHash() + finishedHash := newFinishedHash(versionTLS10) if c.config == nil { c.config = defaultConfig() @@ -97,11 +97,9 @@ func (c *Conn) clientHandshake() os.Error { certs[i] = cert } - // If we don't have a root CA set configured then anything is accepted. - // TODO(rsc): Find certificates for OS X 10.6. - if c.config.RootCAs != nil { + if !c.config.InsecureSkipVerify { opts := x509.VerifyOptions{ - Roots: c.config.RootCAs, + Roots: c.config.rootCAs(), CurrentTime: c.config.time(), DNSName: c.config.ServerName, Intermediates: x509.NewCertPool(), @@ -247,11 +245,11 @@ func (c *Conn) clientHandshake() os.Error { } masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := - keysFromPreMasterSecret10(preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen) + keysFromPreMasterSecret(c.vers, preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen) clientCipher := suite.cipher(clientKey, clientIV, false /* not for reading */ ) - clientHash := suite.mac(clientMAC) - c.out.prepareCipherSpec(clientCipher, clientHash) + clientHash := suite.mac(c.vers, clientMAC) + c.out.prepareCipherSpec(c.vers, clientCipher, clientHash) c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) if serverHello.nextProtoNeg { @@ -271,8 +269,8 @@ func (c *Conn) clientHandshake() os.Error { c.writeRecord(recordTypeHandshake, finished.marshal()) serverCipher := suite.cipher(serverKey, serverIV, true /* for reading */ ) - serverHash := suite.mac(serverMAC) - c.in.prepareCipherSpec(serverCipher, serverHash) + serverHash := suite.mac(c.vers, serverMAC) + c.in.prepareCipherSpec(c.vers, serverCipher, serverHash) c.readRecord(recordTypeChangeCipherSpec) if c.err != nil { return c.err diff --git a/libgo/go/crypto/tls/handshake_client_test.go b/libgo/go/crypto/tls/handshake_client_test.go index 3f91c7a..c0abcda 100644 --- a/libgo/go/crypto/tls/handshake_client_test.go +++ b/libgo/go/crypto/tls/handshake_client_test.go @@ -18,6 +18,7 @@ func testClientScript(t *testing.T, name string, clientScript [][]byte, config * go func() { cli.Write([]byte("hello\n")) cli.Close() + c.Close() }() defer c.Close() diff --git a/libgo/go/crypto/tls/handshake_messages.go b/libgo/go/crypto/tls/handshake_messages.go index 6645adc..f11232d 100644 --- a/libgo/go/crypto/tls/handshake_messages.go +++ b/libgo/go/crypto/tls/handshake_messages.go @@ -676,9 +676,9 @@ func (m *finishedMsg) marshal() (x []byte) { return m.raw } - x = make([]byte, 16) + x = make([]byte, 4+len(m.verifyData)) x[0] = typeFinished - x[3] = 12 + x[3] = byte(len(m.verifyData)) copy(x[4:], m.verifyData) m.raw = x return @@ -686,7 +686,7 @@ func (m *finishedMsg) marshal() (x []byte) { func (m *finishedMsg) unmarshal(data []byte) bool { m.raw = data - if len(data) != 4+12 { + if len(data) < 4 { return false } m.verifyData = data[4:] diff --git a/libgo/go/crypto/tls/handshake_messages_test.go b/libgo/go/crypto/tls/handshake_messages_test.go index 23f729d..dc68a12 100644 --- a/libgo/go/crypto/tls/handshake_messages_test.go +++ b/libgo/go/crypto/tls/handshake_messages_test.go @@ -14,13 +14,13 @@ import ( var tests = []interface{}{ &clientHelloMsg{}, &serverHelloMsg{}, + &finishedMsg{}, &certificateMsg{}, &certificateRequestMsg{}, &certificateVerifyMsg{}, &certificateStatusMsg{}, &clientKeyExchangeMsg{}, - &finishedMsg{}, &nextProtoMsg{}, } @@ -59,11 +59,12 @@ func TestMarshalUnmarshal(t *testing.T) { break } - if i >= 2 { - // The first two message types (ClientHello and - // ServerHello) are allowed to have parsable - // prefixes because the extension data is - // optional. + if i >= 3 { + // The first three message types (ClientHello, + // ServerHello and Finished) are allowed to + // have parsable prefixes because the extension + // data is optional and the length of the + // Finished varies across versions. for j := 0; j < len(marshaled); j++ { if m2.unmarshal(marshaled[0:j]) { t.Errorf("#%d unmarshaled a prefix of length %d of %#v", i, j, m1) diff --git a/libgo/go/crypto/tls/handshake_server.go b/libgo/go/crypto/tls/handshake_server.go index 44a3240..ed9a2e6a 100644 --- a/libgo/go/crypto/tls/handshake_server.go +++ b/libgo/go/crypto/tls/handshake_server.go @@ -30,7 +30,7 @@ func (c *Conn) serverHandshake() os.Error { c.vers = vers c.haveVers = true - finishedHash := newFinishedHash() + finishedHash := newFinishedHash(vers) finishedHash.Write(clientHello.marshal()) hello := new(serverHelloMsg) @@ -115,7 +115,12 @@ FindCipherSuite: } certMsg := new(certificateMsg) - certMsg.certificates = config.Certificates[0].Certificate + if len(clientHello.serverName) > 0 { + c.serverName = clientHello.serverName + certMsg.certificates = config.getCertificateForName(clientHello.serverName).Certificate + } else { + certMsg.certificates = config.Certificates[0].Certificate + } finishedHash.Write(certMsg.marshal()) c.writeRecord(recordTypeHandshake, certMsg.marshal()) @@ -128,7 +133,6 @@ FindCipherSuite: } keyAgreement := suite.ka() - skx, err := keyAgreement.generateServerKeyExchange(config, clientHello, hello) if err != nil { c.sendAlert(alertHandshakeFailure) @@ -235,18 +239,18 @@ FindCipherSuite: finishedHash.Write(certVerify.marshal()) } - preMasterSecret, err := keyAgreement.processClientKeyExchange(config, ckx) + preMasterSecret, err := keyAgreement.processClientKeyExchange(config, ckx, c.vers) if err != nil { c.sendAlert(alertHandshakeFailure) return err } masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := - keysFromPreMasterSecret10(preMasterSecret, clientHello.random, hello.random, suite.macLen, suite.keyLen, suite.ivLen) + keysFromPreMasterSecret(c.vers, preMasterSecret, clientHello.random, hello.random, suite.macLen, suite.keyLen, suite.ivLen) clientCipher := suite.cipher(clientKey, clientIV, true /* for reading */ ) - clientHash := suite.mac(clientMAC) - c.in.prepareCipherSpec(clientCipher, clientHash) + clientHash := suite.mac(c.vers, clientMAC) + c.in.prepareCipherSpec(c.vers, clientCipher, clientHash) c.readRecord(recordTypeChangeCipherSpec) if err := c.error(); err != nil { return err @@ -283,8 +287,8 @@ FindCipherSuite: finishedHash.Write(clientFinished.marshal()) serverCipher := suite.cipher(serverKey, serverIV, false /* not for reading */ ) - serverHash := suite.mac(serverMAC) - c.out.prepareCipherSpec(serverCipher, serverHash) + serverHash := suite.mac(c.vers, serverMAC) + c.out.prepareCipherSpec(c.vers, serverCipher, serverHash) c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) finished := new(finishedMsg) diff --git a/libgo/go/crypto/tls/handshake_server_test.go b/libgo/go/crypto/tls/handshake_server_test.go index b77646e..1939f3d 100644 --- a/libgo/go/crypto/tls/handshake_server_test.go +++ b/libgo/go/crypto/tls/handshake_server_test.go @@ -13,6 +13,8 @@ import ( "io" "net" "os" + "strconv" + "strings" "testing" ) @@ -36,6 +38,7 @@ func init() { testConfig.Certificates[0].Certificate = [][]byte{testCertificate} testConfig.Certificates[0].PrivateKey = testPrivateKey testConfig.CipherSuites = []uint16{TLS_RSA_WITH_RC4_128_SHA} + testConfig.InsecureSkipVerify = true } func testClientHelloFailure(t *testing.T, m handshakeMessage, expected os.Error) { @@ -62,7 +65,7 @@ func TestSimpleError(t *testing.T) { testClientHelloFailure(t, &serverHelloDoneMsg{}, alertUnexpectedMessage) } -var badProtocolVersions = []uint16{0x0000, 0x0005, 0x0100, 0x0105, 0x0200, 0x0205, 0x0300} +var badProtocolVersions = []uint16{0x0000, 0x0005, 0x0100, 0x0105, 0x0200, 0x0205} func TestRejectBadProtocolVersion(t *testing.T) { for _, v := range badProtocolVersions { @@ -112,6 +115,7 @@ func testServerScript(t *testing.T, name string, serverScript [][]byte, config * go func() { srv.Write([]byte("hello, world\n")) srv.Close() + s.Close() }() defer c.Close() @@ -121,9 +125,9 @@ func testServerScript(t *testing.T, name string, serverScript [][]byte, config * continue } bb := make([]byte, len(b)) - _, err := io.ReadFull(c, bb) + n, err := io.ReadFull(c, bb) if err != nil { - t.Fatalf("%s #%d: %s", name, i, err) + t.Fatalf("%s #%d: %s\nRead %d, wanted %d, got %x, wanted %x\n", name, i, err, n, len(bb), bb[:n], b) } if !bytes.Equal(b, bb) { t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) @@ -135,6 +139,13 @@ func TestHandshakeServerRC4(t *testing.T) { testServerScript(t, "RC4", rc4ServerScript, testConfig) } +func TestHandshakeServer3DES(t *testing.T) { + des3Config := new(Config) + *des3Config = *testConfig + des3Config.CipherSuites = []uint16{TLS_RSA_WITH_3DES_EDE_CBC_SHA} + testServerScript(t, "3DES", des3ServerScript, des3Config) +} + func TestHandshakeServerAES(t *testing.T) { aesConfig := new(Config) *aesConfig = *testConfig @@ -142,13 +153,30 @@ func TestHandshakeServerAES(t *testing.T) { testServerScript(t, "AES", aesServerScript, aesConfig) } +func TestHandshakeServerSSLv3(t *testing.T) { + testServerScript(t, "SSLv3", sslv3ServerScript, testConfig) +} + var serve = flag.Bool("serve", false, "run a TLS server on :10443") +var testCipherSuites = flag.String("ciphersuites", + "0x"+strconv.Itob(int(TLS_RSA_WITH_RC4_128_SHA), 16), + "cipher suites to accept in serving mode") func TestRunServer(t *testing.T) { if !*serve { return } + suites := strings.Split(*testCipherSuites, ",") + testConfig.CipherSuites = make([]uint16, len(suites)) + for i := range suites { + suite, err := strconv.Btoui64(suites[i], 0) + if err != nil { + panic(err) + } + testConfig.CipherSuites[i] = uint16(suite) + } + l, err := Listen("tcp", ":10443", testConfig) if err != nil { t.Fatal(err) @@ -356,6 +384,179 @@ var rc4ServerScript = [][]byte{ }, } +var des3ServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7a, 0x01, 0x00, 0x00, + 0x76, 0x03, 0x02, 0x4e, 0x84, 0xf4, 0x3c, 0xe4, + 0xb8, 0xc7, 0xa0, 0x30, 0x55, 0x2a, 0xbc, 0xb7, + 0x04, 0x6b, 0x6f, 0x87, 0x93, 0x96, 0xbd, 0x1a, + 0x7a, 0x1e, 0xce, 0xd2, 0x0d, 0xf3, 0x01, 0x03, + 0xbe, 0x7b, 0x17, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x19, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0xae, 0xcf, 0x4f, 0x70, 0x0e, + 0xe5, 0xe7, 0xba, 0xef, 0x0c, 0x66, 0xe9, 0xae, + 0x76, 0xf4, 0xe0, 0xbc, 0x1c, 0x22, 0x5b, 0x72, + 0xc9, 0x68, 0x63, 0x44, 0xec, 0x72, 0xc2, 0xca, + 0xac, 0xc2, 0xf5, 0x5c, 0x28, 0xa1, 0xaf, 0xd0, + 0xc2, 0xf7, 0x79, 0x71, 0x32, 0x73, 0x86, 0xea, + 0x39, 0xf6, 0x04, 0x26, 0x19, 0x84, 0x1d, 0x7d, + 0xa1, 0x21, 0xa6, 0x88, 0xbf, 0x33, 0x5a, 0x64, + 0xb0, 0xc2, 0xcc, 0x19, 0x7a, 0x8b, 0x6e, 0x94, + 0x9e, 0x2e, 0x20, 0xbe, 0xdc, 0xe9, 0x8e, 0xae, + 0x5c, 0x39, 0xc8, 0xcd, 0x0e, 0x19, 0x9a, 0xa2, + 0xfc, 0x3f, 0x61, 0x9a, 0xca, 0x58, 0x69, 0x0d, + 0xa8, 0x7b, 0xbe, 0x98, 0x8f, 0xb9, 0x9d, 0x8b, + 0x68, 0x65, 0xa9, 0x74, 0xcc, 0x8d, 0x0c, 0xb2, + 0xc4, 0x0f, 0xdc, 0x56, 0x3e, 0x44, 0x61, 0x0a, + 0x26, 0x93, 0x99, 0xef, 0x67, 0xff, 0x6e, 0x73, + 0x01, 0xa1, 0x90, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x60, 0x49, 0x36, + 0xc8, 0x38, 0x95, 0xe4, 0x5d, 0x8e, 0x80, 0x10, + 0x26, 0x9f, 0x87, 0x7d, 0xcd, 0xb9, 0x32, 0x6c, + 0xff, 0xaa, 0xe0, 0x07, 0xec, 0x33, 0xe2, 0x36, + 0x9d, 0xd5, 0x83, 0x2c, 0xf0, 0x0a, 0xa0, 0xa8, + 0x12, 0x9f, 0xca, 0x72, 0xda, 0x70, 0x7d, 0x76, + 0x80, 0x12, 0x88, 0x07, 0xaa, 0x27, 0x62, 0x33, + 0xab, 0x55, 0xad, 0x3c, 0x2b, 0x54, 0xc4, 0x1c, + 0x91, 0xfd, 0x8f, 0x9c, 0xa7, 0x8b, 0x75, 0x10, + 0xa8, 0x6e, 0xfc, 0x30, 0x52, 0x8a, 0x61, 0x02, + 0xdb, 0x9c, 0x6f, 0xc8, 0x19, 0x93, 0x5d, 0x41, + 0x1d, 0x36, 0x68, 0x0b, 0xec, 0x30, 0xae, 0xfb, + 0x90, 0xdb, 0x6d, 0x83, 0xb0, 0xf2, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x28, 0x07, 0xf3, 0x33, 0x84, 0xb1, + 0x5d, 0x2b, 0x52, 0xa4, 0x63, 0x3c, 0x32, 0xe0, + 0x0d, 0x22, 0xf5, 0x23, 0xec, 0xf9, 0xa6, 0xec, + 0xc0, 0x12, 0x69, 0x88, 0xf6, 0x7d, 0x37, 0xcd, + 0xc2, 0x74, 0x2f, 0xef, 0xf6, 0x49, 0x15, 0xea, + 0x88, 0x3f, 0x55, 0x17, 0x03, 0x01, 0x00, 0x28, + 0xaf, 0x00, 0x84, 0xff, 0x11, 0x01, 0x6d, 0xba, + 0x39, 0x5e, 0x45, 0xe1, 0x52, 0x5e, 0xc1, 0xab, + 0xde, 0x5b, 0x16, 0xdd, 0xd6, 0x61, 0x57, 0xb8, + 0x66, 0x8b, 0x2d, 0xde, 0x51, 0x41, 0xc5, 0x09, + 0xb3, 0x6a, 0x06, 0x43, 0xb4, 0x73, 0x5c, 0xf1, + 0x15, 0x03, 0x01, 0x00, 0x18, 0xbd, 0x65, 0xb2, + 0xce, 0x77, 0x2e, 0xf9, 0x11, 0xc4, 0x80, 0x43, + 0x5a, 0x73, 0x8b, 0x73, 0xdd, 0xf0, 0x54, 0x44, + 0x7c, 0x56, 0x19, 0x54, 0xda, + }, +} + var aesServerScript = [][]byte{ { 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, @@ -515,3 +716,165 @@ var aesServerScript = [][]byte{ 0xcd, 0x84, 0xf0, }, } + +var sslv3ServerScript = [][]byte{ + { + 0x16, 0x03, 0x00, 0x00, 0x41, 0x01, 0x00, 0x00, + 0x3d, 0x03, 0x00, 0x4e, 0x70, 0xe2, 0x18, 0x86, + 0xd6, 0xc6, 0x6f, 0xf3, 0xc8, 0xf4, 0x02, 0xd6, + 0x4d, 0xee, 0x17, 0x32, 0x4b, 0xd2, 0x78, 0xd8, + 0xa1, 0x03, 0x5d, 0x68, 0x82, 0x89, 0xbe, 0xfd, + 0x12, 0xb9, 0x06, 0x00, 0x00, 0x16, 0x00, 0x33, + 0x00, 0x39, 0x00, 0x16, 0x00, 0x32, 0x00, 0x38, + 0x00, 0x13, 0x00, 0x2f, 0x00, 0x35, 0x00, 0x0a, + 0x00, 0x05, 0x00, 0x04, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x00, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x00, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x00, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x00, 0x00, 0x84, 0x10, 0x00, 0x00, + 0x80, 0x74, 0x0e, 0x3a, 0xcf, 0xba, 0x9f, 0x1a, + 0x9b, 0xb2, 0xa4, 0xc7, 0x5d, 0xf3, 0x0c, 0x80, + 0x06, 0x80, 0xf3, 0x57, 0xb2, 0xd9, 0x36, 0x24, + 0x6a, 0x06, 0x13, 0x40, 0xf9, 0x7c, 0xb9, 0x3e, + 0x4b, 0x68, 0x4f, 0x21, 0x90, 0x2d, 0xbd, 0xca, + 0xd4, 0x83, 0xf0, 0x7a, 0xeb, 0x7a, 0x74, 0x1b, + 0xcd, 0xfe, 0x69, 0xef, 0xc0, 0x86, 0xa0, 0x24, + 0x31, 0x65, 0x40, 0xd2, 0xdd, 0x6f, 0xb9, 0xd7, + 0x8d, 0xc1, 0x69, 0x60, 0x44, 0x7a, 0x75, 0xfb, + 0x42, 0x6a, 0x0f, 0x66, 0x45, 0x10, 0x73, 0xee, + 0x87, 0x28, 0x37, 0x83, 0x86, 0xd8, 0x5a, 0xc8, + 0x60, 0x87, 0xda, 0x33, 0x87, 0xaf, 0x34, 0x8b, + 0xf5, 0x61, 0x63, 0x7a, 0x5c, 0x60, 0x26, 0xb9, + 0xdb, 0xa1, 0xb7, 0xe3, 0x60, 0x38, 0x94, 0x5c, + 0x83, 0x23, 0xd6, 0x8d, 0xc2, 0x14, 0x4a, 0x0f, + 0x0e, 0x4f, 0xf9, 0x4e, 0x7b, 0x15, 0xcd, 0x18, + 0x04, 0x14, 0x03, 0x00, 0x00, 0x01, 0x01, 0x16, + 0x03, 0x00, 0x00, 0x3c, 0xbd, 0xbc, 0xec, 0xdc, + 0x79, 0xb1, 0xae, 0x16, 0xc9, 0x26, 0x9a, 0xc0, + 0xc0, 0x2c, 0x33, 0x36, 0x13, 0x91, 0x58, 0x5d, + 0x7d, 0xee, 0x4e, 0xd8, 0x7e, 0xac, 0x88, 0x87, + 0x0a, 0x75, 0x66, 0xb1, 0x44, 0x79, 0x2f, 0x42, + 0xe8, 0x92, 0x74, 0x4c, 0xab, 0x36, 0xc8, 0x17, + 0x5f, 0x02, 0x8a, 0x20, 0x53, 0xe9, 0x1d, 0xb4, + 0xfe, 0x5c, 0x2b, 0xd9, 0x0a, 0xfb, 0xc6, 0x63, + }, + + { + 0x14, 0x03, 0x00, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x00, 0x00, 0x3c, 0xaa, 0xa1, 0x98, 0xc4, 0x6b, + 0x5a, 0x16, 0x3f, 0x5f, 0xa4, 0x96, 0x3e, 0x78, + 0xe4, 0x6f, 0x49, 0x05, 0x47, 0xc4, 0x05, 0x60, + 0xeb, 0x0b, 0x45, 0xe3, 0xbc, 0x50, 0x11, 0x24, + 0x5f, 0x01, 0xd7, 0xb8, 0x8f, 0x60, 0x63, 0x66, + 0xbd, 0x3e, 0xd9, 0xa8, 0x80, 0x43, 0x9f, 0x0b, + 0x51, 0x61, 0xed, 0x13, 0xc6, 0x21, 0xd0, 0xfe, + 0xbc, 0x17, 0x3c, 0x36, 0xb0, 0x82, 0x7f, 0x17, + 0x03, 0x00, 0x00, 0x21, 0xee, 0x44, 0xf3, 0xa6, + 0x88, 0x9d, 0x78, 0x44, 0xde, 0xdf, 0xeb, 0xc5, + 0xad, 0xc4, 0xcc, 0x56, 0x5c, 0x54, 0x96, 0x52, + 0x3f, 0xd9, 0x40, 0x6e, 0x79, 0xd8, 0x58, 0x78, + 0x4f, 0x5a, 0xe9, 0x06, 0xef, 0x15, 0x03, 0x00, + 0x00, 0x16, 0xd3, 0xc2, 0x52, 0x99, 0x2a, 0x84, + 0xc4, 0x52, 0x5f, 0x3b, 0x19, 0xe7, 0xfc, 0x65, + 0xaf, 0xd3, 0xb7, 0xa3, 0xcc, 0x4a, 0x1d, 0x2e, + }, +} diff --git a/libgo/go/crypto/tls/key_agreement.go b/libgo/go/crypto/tls/key_agreement.go index a40d18f..e347528 100644 --- a/libgo/go/crypto/tls/key_agreement.go +++ b/libgo/go/crypto/tls/key_agreement.go @@ -24,7 +24,7 @@ func (ka rsaKeyAgreement) generateServerKeyExchange(config *Config, clientHello return nil, nil } -func (ka rsaKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { +func (ka rsaKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg, version uint16) ([]byte, os.Error) { preMasterSecret := make([]byte, 48) _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) if err != nil { @@ -34,11 +34,15 @@ func (ka rsaKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKe if len(ckx.ciphertext) < 2 { return nil, os.NewError("bad ClientKeyExchange") } - ciphertextLen := int(ckx.ciphertext[0])<<8 | int(ckx.ciphertext[1]) - if ciphertextLen != len(ckx.ciphertext)-2 { - return nil, os.NewError("bad ClientKeyExchange") + + ciphertext := ckx.ciphertext + if version != versionSSL30 { + ciphertextLen := int(ckx.ciphertext[0])<<8 | int(ckx.ciphertext[1]) + if ciphertextLen != len(ckx.ciphertext)-2 { + return nil, os.NewError("bad ClientKeyExchange") + } + ciphertext = ckx.ciphertext[2:] } - ciphertext := ckx.ciphertext[2:] err = rsa.DecryptPKCS1v15SessionKey(config.rand(), config.Certificates[0].PrivateKey, ciphertext, preMasterSecret) if err != nil { @@ -159,7 +163,7 @@ Curve: return skx, nil } -func (ka *ecdheRSAKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { +func (ka *ecdheRSAKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg, version uint16) ([]byte, os.Error) { if len(ckx.ciphertext) == 0 || int(ckx.ciphertext[0]) != len(ckx.ciphertext)-1 { return nil, os.NewError("bad ClientKeyExchange") } diff --git a/libgo/go/crypto/tls/prf.go b/libgo/go/crypto/tls/prf.go index 478cf65..2d58dc5 100644 --- a/libgo/go/crypto/tls/prf.go +++ b/libgo/go/crypto/tls/prf.go @@ -63,6 +63,39 @@ func pRF10(result, secret, label, seed []byte) { } } +// pRF30 implements the SSL 3.0 pseudo-random function, as defined in +// www.mozilla.org/projects/security/pki/nss/ssl/draft302.txt section 6. +func pRF30(result, secret, label, seed []byte) { + hashSHA1 := sha1.New() + hashMD5 := md5.New() + + done := 0 + i := 0 + // RFC5246 section 6.3 says that the largest PRF output needed is 128 + // bytes. Since no more ciphersuites will be added to SSLv3, this will + // remain true. Each iteration gives us 16 bytes so 10 iterations will + // be sufficient. + var b [11]byte + for done < len(result) { + for j := 0; j <= i; j++ { + b[j] = 'A' + byte(i) + } + + hashSHA1.Reset() + hashSHA1.Write(b[:i+1]) + hashSHA1.Write(secret) + hashSHA1.Write(seed) + digest := hashSHA1.Sum() + + hashMD5.Reset() + hashMD5.Write(secret) + hashMD5.Write(digest) + + done += copy(result[done:], hashMD5.Sum()) + i++ + } +} + const ( tlsRandomLength = 32 // Length of a random nonce in TLS 1.1. masterSecretLength = 48 // Length of a master secret in TLS 1.1. @@ -77,19 +110,24 @@ var serverFinishedLabel = []byte("server finished") // keysFromPreMasterSecret generates the connection keys from the pre master // secret, given the lengths of the MAC key, cipher key and IV, as defined in // RFC 2246, section 6.3. -func keysFromPreMasterSecret10(preMasterSecret, clientRandom, serverRandom []byte, macLen, keyLen, ivLen int) (masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV []byte) { +func keysFromPreMasterSecret(version uint16, preMasterSecret, clientRandom, serverRandom []byte, macLen, keyLen, ivLen int) (masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV []byte) { + prf := pRF10 + if version == versionSSL30 { + prf = pRF30 + } + var seed [tlsRandomLength * 2]byte copy(seed[0:len(clientRandom)], clientRandom) copy(seed[len(clientRandom):], serverRandom) masterSecret = make([]byte, masterSecretLength) - pRF10(masterSecret, preMasterSecret, masterSecretLabel, seed[0:]) + prf(masterSecret, preMasterSecret, masterSecretLabel, seed[0:]) copy(seed[0:len(clientRandom)], serverRandom) copy(seed[len(serverRandom):], clientRandom) n := 2*macLen + 2*keyLen + 2*ivLen keyMaterial := make([]byte, n) - pRF10(keyMaterial, masterSecret, keyExpansionLabel, seed[0:]) + prf(keyMaterial, masterSecret, keyExpansionLabel, seed[0:]) clientMAC = keyMaterial[:macLen] keyMaterial = keyMaterial[macLen:] serverMAC = keyMaterial[:macLen] @@ -104,6 +142,10 @@ func keysFromPreMasterSecret10(preMasterSecret, clientRandom, serverRandom []byt return } +func newFinishedHash(version uint16) finishedHash { + return finishedHash{md5.New(), sha1.New(), md5.New(), sha1.New(), version} +} + // A finishedHash calculates the hash of a set of handshake messages suitable // for including in a Finished message. type finishedHash struct { @@ -111,10 +153,7 @@ type finishedHash struct { clientSHA1 hash.Hash serverMD5 hash.Hash serverSHA1 hash.Hash -} - -func newFinishedHash() finishedHash { - return finishedHash{md5.New(), sha1.New(), md5.New(), sha1.New()} + version uint16 } func (h finishedHash) Write(msg []byte) (n int, err os.Error) { @@ -125,9 +164,10 @@ func (h finishedHash) Write(msg []byte) (n int, err os.Error) { return len(msg), nil } -// finishedSum calculates the contents of the verify_data member of a Finished -// message given the MD5 and SHA1 hashes of a set of handshake messages. -func finishedSum(md5, sha1, label, masterSecret []byte) []byte { +// finishedSum10 calculates the contents of the verify_data member of a TLSv1 +// Finished message given the MD5 and SHA1 hashes of a set of handshake +// messages. +func finishedSum10(md5, sha1, label, masterSecret []byte) []byte { seed := make([]byte, len(md5)+len(sha1)) copy(seed, md5) copy(seed[len(md5):], sha1) @@ -136,18 +176,61 @@ func finishedSum(md5, sha1, label, masterSecret []byte) []byte { return out } +// finishedSum30 calculates the contents of the verify_data member of a SSLv3 +// Finished message given the MD5 and SHA1 hashes of a set of handshake +// messages. +func finishedSum30(md5, sha1 hash.Hash, masterSecret []byte, magic [4]byte) []byte { + md5.Write(magic[:]) + md5.Write(masterSecret) + md5.Write(ssl30Pad1[:]) + md5Digest := md5.Sum() + + md5.Reset() + md5.Write(masterSecret) + md5.Write(ssl30Pad2[:]) + md5.Write(md5Digest) + md5Digest = md5.Sum() + + sha1.Write(magic[:]) + sha1.Write(masterSecret) + sha1.Write(ssl30Pad1[:40]) + sha1Digest := sha1.Sum() + + sha1.Reset() + sha1.Write(masterSecret) + sha1.Write(ssl30Pad2[:40]) + sha1.Write(sha1Digest) + sha1Digest = sha1.Sum() + + ret := make([]byte, len(md5Digest)+len(sha1Digest)) + copy(ret, md5Digest) + copy(ret[len(md5Digest):], sha1Digest) + return ret +} + +var ssl3ClientFinishedMagic = [4]byte{0x43, 0x4c, 0x4e, 0x54} +var ssl3ServerFinishedMagic = [4]byte{0x53, 0x52, 0x56, 0x52} + // clientSum returns the contents of the verify_data member of a client's // Finished message. func (h finishedHash) clientSum(masterSecret []byte) []byte { + if h.version == versionSSL30 { + return finishedSum30(h.clientMD5, h.clientSHA1, masterSecret, ssl3ClientFinishedMagic) + } + md5 := h.clientMD5.Sum() sha1 := h.clientSHA1.Sum() - return finishedSum(md5, sha1, clientFinishedLabel, masterSecret) + return finishedSum10(md5, sha1, clientFinishedLabel, masterSecret) } // serverSum returns the contents of the verify_data member of a server's // Finished message. func (h finishedHash) serverSum(masterSecret []byte) []byte { + if h.version == versionSSL30 { + return finishedSum30(h.serverMD5, h.serverSHA1, masterSecret, ssl3ServerFinishedMagic) + } + md5 := h.serverMD5.Sum() sha1 := h.serverSHA1.Sum() - return finishedSum(md5, sha1, serverFinishedLabel, masterSecret) + return finishedSum10(md5, sha1, serverFinishedLabel, masterSecret) } diff --git a/libgo/go/crypto/tls/prf_test.go b/libgo/go/crypto/tls/prf_test.go index f8c4acb..a32392c 100644 --- a/libgo/go/crypto/tls/prf_test.go +++ b/libgo/go/crypto/tls/prf_test.go @@ -34,6 +34,7 @@ func TestSplitPreMasterSecret(t *testing.T) { } type testKeysFromTest struct { + version uint16 preMasterSecret string clientRandom, serverRandom string masterSecret string @@ -47,7 +48,7 @@ func TestKeysFromPreMasterSecret(t *testing.T) { in, _ := hex.DecodeString(test.preMasterSecret) clientRandom, _ := hex.DecodeString(test.clientRandom) serverRandom, _ := hex.DecodeString(test.serverRandom) - master, clientMAC, serverMAC, clientKey, serverKey, _, _ := keysFromPreMasterSecret10(in, clientRandom, serverRandom, test.macLen, test.keyLen, 0) + master, clientMAC, serverMAC, clientKey, serverKey, _, _ := keysFromPreMasterSecret(test.version, in, clientRandom, serverRandom, test.macLen, test.keyLen, 0) masterString := hex.EncodeToString(master) clientMACString := hex.EncodeToString(clientMAC) serverMACString := hex.EncodeToString(serverMAC) @@ -58,7 +59,7 @@ func TestKeysFromPreMasterSecret(t *testing.T) { serverMACString != test.serverMAC || clientKeyString != test.clientKey || serverKeyString != test.serverKey { - t.Errorf("#%d: got: (%s, %s, %s, %s, %s) want: (%s, %s, %s, %s %s)", i, masterString, clientMACString, serverMACString, clientKeyString, serverMACString, test.masterSecret, test.clientMAC, test.serverMAC, test.clientKey, test.serverKey) + t.Errorf("#%d: got: (%s, %s, %s, %s, %s) want: (%s, %s, %s, %s, %s)", i, masterString, clientMACString, serverMACString, clientKeyString, serverKeyString, test.masterSecret, test.clientMAC, test.serverMAC, test.clientKey, test.serverKey) } } } @@ -66,6 +67,7 @@ func TestKeysFromPreMasterSecret(t *testing.T) { // These test vectors were generated from GnuTLS using `gnutls-cli --insecure -d 9 ` var testKeysFromTests = []testKeysFromTest{ { + versionTLS10, "0302cac83ad4b1db3b9ab49ad05957de2a504a634a386fc600889321e1a971f57479466830ac3e6f468e87f5385fa0c5", "4ae66303755184a3917fcb44880605fcc53baa01912b22ed94473fc69cebd558", "4ae663020ec16e6bb5130be918cfcafd4d765979a3136a5d50c593446e4e44db", @@ -78,6 +80,7 @@ var testKeysFromTests = []testKeysFromTest{ 16, }, { + versionTLS10, "03023f7527316bc12cbcd69e4b9e8275d62c028f27e65c745cfcddc7ce01bd3570a111378b63848127f1c36e5f9e4890", "4ae66364b5ea56b20ce4e25555aed2d7e67f42788dd03f3fee4adae0459ab106", "4ae66363ab815cbf6a248b87d6b556184e945e9b97fbdf247858b0bdafacfa1c", @@ -90,6 +93,7 @@ var testKeysFromTests = []testKeysFromTest{ 16, }, { + versionTLS10, "832d515f1d61eebb2be56ba0ef79879efb9b527504abb386fb4310ed5d0e3b1f220d3bb6b455033a2773e6d8bdf951d278a187482b400d45deb88a5d5a6bb7d6a7a1decc04eb9ef0642876cd4a82d374d3b6ff35f0351dc5d411104de431375355addc39bfb1f6329fb163b0bc298d658338930d07d313cd980a7e3d9196cac1", "4ae663b2ee389c0de147c509d8f18f5052afc4aaf9699efe8cb05ece883d3a5e", "4ae664d503fd4cff50cfc1fb8fc606580f87b0fcdac9554ba0e01d785bdf278e", @@ -101,4 +105,17 @@ var testKeysFromTests = []testKeysFromTest{ 20, 16, }, + { + versionSSL30, + "832d515f1d61eebb2be56ba0ef79879efb9b527504abb386fb4310ed5d0e3b1f220d3bb6b455033a2773e6d8bdf951d278a187482b400d45deb88a5d5a6bb7d6a7a1decc04eb9ef0642876cd4a82d374d3b6ff35f0351dc5d411104de431375355addc39bfb1f6329fb163b0bc298d658338930d07d313cd980a7e3d9196cac1", + "4ae663b2ee389c0de147c509d8f18f5052afc4aaf9699efe8cb05ece883d3a5e", + "4ae664d503fd4cff50cfc1fb8fc606580f87b0fcdac9554ba0e01d785bdf278e", + "a614863e56299dcffeea2938f22c2ba023768dbe4b3f6877bc9c346c6ae529b51d9cb87ff9695ea4d01f2205584405b2", + "2c450d5b6f6e2013ac6bea6a0b32200d4e1ffb94", + "7a7a7438769536f2fb1ae49a61f0703b79b2dc53", + "f8f6b26c10f12855c9aafb1e0e839ccf", + "2b9d4b4a60cb7f396780ebff50650419", + 20, + 16, + }, } diff --git a/libgo/go/crypto/tls/root_darwin.go b/libgo/go/crypto/tls/root_darwin.go new file mode 100644 index 0000000..1512241 --- /dev/null +++ b/libgo/go/crypto/tls/root_darwin.go @@ -0,0 +1,95 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +/* +// Note: We disable -Werror here because the code in this file uses a deprecated API to stay +// compatible with both Mac OS X 10.6 and 10.7. Using a deprecated function on Darwin generates +// a warning. +#cgo CFLAGS: -Wno-error +#cgo LDFLAGS: -framework CoreFoundation -framework Security +#include <CoreFoundation/CoreFoundation.h> +#include <Security/Security.h> + +// FetchPEMRoots fetches the system's list of trusted X.509 root certificates. +// +// On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root +// certificates of the system. On failure, the function returns -1. +// +// Note: The CFDataRef returned in pemRoots must be released (using CFRelease) after +// we've consumed its content. +int FetchPEMRoots(CFDataRef *pemRoots) { + if (pemRoots == NULL) { + return -1; + } + + CFArrayRef certs = NULL; + OSStatus err = SecTrustCopyAnchorCertificates(&certs); + if (err != noErr) { + return -1; + } + + CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0); + int i, ncerts = CFArrayGetCount(certs); + for (i = 0; i < ncerts; i++) { + CFDataRef data = NULL; + SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, i); + if (cert == NULL) { + continue; + } + + // SecKeychainImportExport is deprecated in >= OS X 10.7, and has been replaced by + // SecItemExport. If we're built on a host with a Lion SDK, this code gets conditionally + // included in the output, also for binaries meant for 10.6. + // + // To make sure that we run on both Mac OS X 10.6 and 10.7 we use weak linking + // and check whether SecItemExport is available before we attempt to call it. On + // 10.6, this won't be the case, and we'll fall back to calling SecKeychainItemExport. +#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 + if (SecItemExport) { + err = SecItemExport(cert, kSecFormatX509Cert, kSecItemPemArmour, NULL, &data); + if (err != noErr) { + continue; + } + } else +#endif + if (data == NULL) { + err = SecKeychainItemExport(cert, kSecFormatX509Cert, kSecItemPemArmour, NULL, &data); + if (err != noErr) { + continue; + } + } + + if (data != NULL) { + CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data)); + CFRelease(data); + } + } + + CFRelease(certs); + + *pemRoots = combinedData; + return 0; +} +*/ +import "C" +import ( + "crypto/x509" + "unsafe" +) + +func initDefaultRoots() { + roots := x509.NewCertPool() + + var data C.CFDataRef = nil + err := C.FetchPEMRoots(&data) + if err != -1 { + defer C.CFRelease(C.CFTypeRef(data)) + buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data))) + roots.AppendCertsFromPEM(buf) + } + + varDefaultRoots = roots +} diff --git a/libgo/go/crypto/tls/root_stub.go b/libgo/go/crypto/tls/root_stub.go new file mode 100644 index 0000000..1903eed --- /dev/null +++ b/libgo/go/crypto/tls/root_stub.go @@ -0,0 +1,8 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +func initDefaultRoots() { +} diff --git a/libgo/go/crypto/tls/root_test.go b/libgo/go/crypto/tls/root_test.go new file mode 100644 index 0000000..95a89d8 --- /dev/null +++ b/libgo/go/crypto/tls/root_test.go @@ -0,0 +1,36 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "testing" +) + +var tlsServers = []string{ + "google.com:443", + "github.com:443", + "twitter.com:443", +} + +func TestOSCertBundles(t *testing.T) { + defaultRoots() + + if testing.Short() { + t.Logf("skipping certificate tests in short mode") + return + } + + for _, addr := range tlsServers { + conn, err := Dial("tcp", addr, nil) + if err != nil { + t.Errorf("unable to verify %v: %v", addr, err) + continue + } + err = conn.Close() + if err != nil { + t.Error(err) + } + } +} diff --git a/libgo/go/crypto/tls/root_unix.go b/libgo/go/crypto/tls/root_unix.go new file mode 100644 index 0000000..095beec --- /dev/null +++ b/libgo/go/crypto/tls/root_unix.go @@ -0,0 +1,29 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/x509" + "io/ioutil" +) + +// Possible certificate files; stop after finding one. +var certFiles = []string{ + "/etc/ssl/certs/ca-certificates.crt", // Linux etc + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL + "/etc/ssl/ca-bundle.pem", // OpenSUSE +} + +func initDefaultRoots() { + roots := x509.NewCertPool() + for _, file := range certFiles { + data, err := ioutil.ReadFile(file) + if err == nil { + roots.AppendCertsFromPEM(data) + break + } + } + varDefaultRoots = roots +} diff --git a/libgo/go/crypto/tls/root_windows.go b/libgo/go/crypto/tls/root_windows.go new file mode 100644 index 0000000..b8e27a9 --- /dev/null +++ b/libgo/go/crypto/tls/root_windows.go @@ -0,0 +1,54 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/x509" + "reflect" + "syscall" + "unsafe" +) + +func loadStore(roots *x509.CertPool, name string) { + store, errno := syscall.CertOpenSystemStore(syscall.InvalidHandle, syscall.StringToUTF16Ptr(name)) + if errno != 0 { + return + } + + var cert *syscall.CertContext + for { + cert = syscall.CertEnumCertificatesInStore(store, cert) + if cert == nil { + break + } + + var asn1Slice []byte + hdrp := (*reflect.SliceHeader)(unsafe.Pointer(&asn1Slice)) + hdrp.Data = cert.EncodedCert + hdrp.Len = int(cert.Length) + hdrp.Cap = int(cert.Length) + + buf := make([]byte, len(asn1Slice)) + copy(buf, asn1Slice) + + if cert, err := x509.ParseCertificate(buf); err == nil { + roots.AddCert(cert) + } + } + + syscall.CertCloseStore(store, 0) +} + +func initDefaultRoots() { + roots := x509.NewCertPool() + + // Roots + loadStore(roots, "ROOT") + + // Intermediates + loadStore(roots, "CA") + + varDefaultRoots = roots +} diff --git a/libgo/go/crypto/x509/cert_pool.go b/libgo/go/crypto/x509/cert_pool.go index 16cd92e..b9196ed 100644 --- a/libgo/go/crypto/x509/cert_pool.go +++ b/libgo/go/crypto/x509/cert_pool.go @@ -5,9 +5,7 @@ package x509 import ( - "crypto/x509/pkix" "encoding/pem" - "strings" ) // Roots is a set of certificates. @@ -26,10 +24,6 @@ func NewCertPool() *CertPool { } } -func nameToKey(name *pkix.Name) string { - return strings.Join(name.Country, ",") + "/" + strings.Join(name.Organization, ",") + "/" + strings.Join(name.OrganizationalUnit, ",") + "/" + name.CommonName -} - // findVerifiedParents attempts to find certificates in s which have signed the // given certificate. If no such certificate can be found or the signature // doesn't match, it returns nil. @@ -40,7 +34,7 @@ func (s *CertPool) findVerifiedParents(cert *Certificate) (parents []int) { candidates = s.bySubjectKeyId[string(cert.AuthorityKeyId)] } if len(candidates) == 0 { - candidates = s.byName[nameToKey(&cert.Issuer)] + candidates = s.byName[string(cert.RawIssuer)] } for _, c := range candidates { @@ -72,7 +66,7 @@ func (s *CertPool) AddCert(cert *Certificate) { keyId := string(cert.SubjectKeyId) s.bySubjectKeyId[keyId] = append(s.bySubjectKeyId[keyId], n) } - name := nameToKey(&cert.Subject) + name := string(cert.RawSubject) s.byName[name] = append(s.byName[name], n) } diff --git a/libgo/go/crypto/x509/pkcs1.go b/libgo/go/crypto/x509/pkcs1.go new file mode 100644 index 0000000..42afcb4 --- /dev/null +++ b/libgo/go/crypto/x509/pkcs1.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "asn1" + "big" + "os" + "crypto/rsa" +) + +// pkcs1PrivateKey is a structure which mirrors the PKCS#1 ASN.1 for an RSA private key. +type pkcs1PrivateKey struct { + Version int + N *big.Int + E int + D *big.Int + P *big.Int + Q *big.Int + // We ignore these values, if present, because rsa will calculate them. + Dp *big.Int `asn1:"optional"` + Dq *big.Int `asn1:"optional"` + Qinv *big.Int `asn1:"optional"` + + AdditionalPrimes []pkcs1AdditionalRSAPrime `asn1:"optional"` +} + +type pkcs1AdditionalRSAPrime struct { + Prime *big.Int + + // We ignore these values because rsa will calculate them. + Exp *big.Int + Coeff *big.Int +} + +// ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. +func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { + var priv pkcs1PrivateKey + rest, err := asn1.Unmarshal(der, &priv) + if len(rest) > 0 { + err = asn1.SyntaxError{"trailing data"} + return + } + if err != nil { + return + } + + if priv.Version > 1 { + return nil, os.NewError("x509: unsupported private key version") + } + + if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 { + return nil, os.NewError("private key contains zero or negative value") + } + + key = new(rsa.PrivateKey) + key.PublicKey = rsa.PublicKey{ + E: priv.E, + N: priv.N, + } + + key.D = priv.D + key.Primes = make([]*big.Int, 2+len(priv.AdditionalPrimes)) + key.Primes[0] = priv.P + key.Primes[1] = priv.Q + for i, a := range priv.AdditionalPrimes { + if a.Prime.Sign() <= 0 { + return nil, os.NewError("private key contains zero or negative prime") + } + key.Primes[i+2] = a.Prime + // We ignore the other two values because rsa will calculate + // them as needed. + } + + err = key.Validate() + if err != nil { + return nil, err + } + key.Precompute() + + return +} + +// MarshalPKCS1PrivateKey converts a private key to ASN.1 DER encoded form. +func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { + key.Precompute() + + version := 0 + if len(key.Primes) > 2 { + version = 1 + } + + priv := pkcs1PrivateKey{ + Version: version, + N: key.N, + E: key.PublicKey.E, + D: key.D, + P: key.Primes[0], + Q: key.Primes[1], + Dp: key.Precomputed.Dp, + Dq: key.Precomputed.Dq, + Qinv: key.Precomputed.Qinv, + } + + priv.AdditionalPrimes = make([]pkcs1AdditionalRSAPrime, len(key.Precomputed.CRTValues)) + for i, values := range key.Precomputed.CRTValues { + priv.AdditionalPrimes[i].Prime = key.Primes[2+i] + priv.AdditionalPrimes[i].Exp = values.Exp + priv.AdditionalPrimes[i].Coeff = values.Coeff + } + + b, _ := asn1.Marshal(priv) + return b +} + +// rsaPublicKey reflects the ASN.1 structure of a PKCS#1 public key. +type rsaPublicKey struct { + N *big.Int + E int +} diff --git a/libgo/go/crypto/x509/pkix/pkix.go b/libgo/go/crypto/x509/pkix/pkix.go index 266fd55..332a394 100644 --- a/libgo/go/crypto/x509/pkix/pkix.go +++ b/libgo/go/crypto/x509/pkix/pkix.go @@ -43,6 +43,8 @@ type Name struct { Locality, Province []string StreetAddress, PostalCode []string SerialNumber, CommonName string + + Names []AttributeTypeAndValue } func (n *Name) FillFromRDNSequence(rdns *RDNSequence) { @@ -51,6 +53,7 @@ func (n *Name) FillFromRDNSequence(rdns *RDNSequence) { continue } atv := rdn[0] + n.Names = append(n.Names, atv) value, ok := atv.Value.(string) if !ok { continue diff --git a/libgo/go/crypto/x509/verify_test.go b/libgo/go/crypto/x509/verify_test.go index 111f60e..eaa8169 100644 --- a/libgo/go/crypto/x509/verify_test.go +++ b/libgo/go/crypto/x509/verify_test.go @@ -5,6 +5,7 @@ package x509 import ( + "crypto/x509/pkix" "encoding/pem" "os" "strings" @@ -31,7 +32,7 @@ var verifyTests = []verifyTest{ dnsName: "www.google.com", expectedChains: [][]string{ - []string{"Google", "Thawte", "VeriSign"}, + {"Google", "Thawte", "VeriSign"}, }, }, { @@ -68,7 +69,7 @@ var verifyTests = []verifyTest{ dnsName: "www.google.com", expectedChains: [][]string{ - []string{"Google", "Thawte", "VeriSign"}, + {"Google", "Thawte", "VeriSign"}, }, }, { @@ -78,7 +79,7 @@ var verifyTests = []verifyTest{ currentTime: 1302726541, expectedChains: [][]string{ - []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, }, }, { @@ -88,8 +89,8 @@ var verifyTests = []verifyTest{ currentTime: 1302726541, expectedChains: [][]string{ - []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, - []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, + {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, }, }, } @@ -211,6 +212,10 @@ func chainToDebugString(chain []*Certificate) string { return chainStr } +func nameToKey(name *pkix.Name) string { + return strings.Join(name.Country, ",") + "/" + strings.Join(name.Organization, ",") + "/" + strings.Join(name.OrganizationalUnit, ",") + "/" + name.CommonName +} + const verisignRoot = `-----BEGIN CERTIFICATE----- MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz diff --git a/libgo/go/crypto/x509/x509.go b/libgo/go/crypto/x509/x509.go index 8fda471..4b8ecc5 100644 --- a/libgo/go/crypto/x509/x509.go +++ b/libgo/go/crypto/x509/x509.go @@ -20,108 +20,59 @@ import ( "time" ) -// pkcs1PrivateKey is a structure which mirrors the PKCS#1 ASN.1 for an RSA private key. -type pkcs1PrivateKey struct { - Version int - N *big.Int - E int - D *big.Int - P *big.Int - Q *big.Int - // We ignore these values, if present, because rsa will calculate them. - Dp *big.Int `asn1:"optional"` - Dq *big.Int `asn1:"optional"` - Qinv *big.Int `asn1:"optional"` - - AdditionalPrimes []pkcs1AdditionalRSAPrime `asn1:"optional"` +// pkixPublicKey reflects a PKIX public key structure. See SubjectPublicKeyInfo +// in RFC 3280. +type pkixPublicKey struct { + Algo pkix.AlgorithmIdentifier + BitString asn1.BitString } -type pkcs1AdditionalRSAPrime struct { - Prime *big.Int - - // We ignore these values because rsa will calculate them. - Exp *big.Int - Coeff *big.Int -} - -// ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. -func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { - var priv pkcs1PrivateKey - rest, err := asn1.Unmarshal(der, &priv) - if len(rest) > 0 { - err = asn1.SyntaxError{"trailing data"} - return - } - if err != nil { +// ParsePKIXPublicKey parses a DER encoded public key. These values are +// typically found in PEM blocks with "BEGIN PUBLIC KEY". +func ParsePKIXPublicKey(derBytes []byte) (pub interface{}, err os.Error) { + var pki publicKeyInfo + if _, err = asn1.Unmarshal(derBytes, &pki); err != nil { return } - - if priv.Version > 1 { - return nil, os.NewError("x509: unsupported private key version") - } - - if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 { - return nil, os.NewError("private key contains zero or negative value") - } - - key = new(rsa.PrivateKey) - key.PublicKey = rsa.PublicKey{ - E: priv.E, - N: priv.N, + algo := getPublicKeyAlgorithmFromOID(pki.Algorithm.Algorithm) + if algo == UnknownPublicKeyAlgorithm { + return nil, os.NewError("ParsePKIXPublicKey: unknown public key algorithm") } - - key.D = priv.D - key.Primes = make([]*big.Int, 2+len(priv.AdditionalPrimes)) - key.Primes[0] = priv.P - key.Primes[1] = priv.Q - for i, a := range priv.AdditionalPrimes { - if a.Prime.Sign() <= 0 { - return nil, os.NewError("private key contains zero or negative prime") - } - key.Primes[i+2] = a.Prime - // We ignore the other two values because rsa will calculate - // them as needed. - } - - err = key.Validate() - if err != nil { - return nil, err - } - key.Precompute() - - return + return parsePublicKey(algo, &pki) } -// MarshalPKCS1PrivateKey converts a private key to ASN.1 DER encoded form. -func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { - key.Precompute() - - version := 0 - if len(key.Primes) > 2 { - version = 1 - } +// MarshalPKIXPublicKey serialises a public key to DER-encoded PKIX format. +func MarshalPKIXPublicKey(pub interface{}) ([]byte, os.Error) { + var pubBytes []byte - priv := pkcs1PrivateKey{ - Version: version, - N: key.N, - E: key.PublicKey.E, - D: key.D, - P: key.Primes[0], - Q: key.Primes[1], - Dp: key.Precomputed.Dp, - Dq: key.Precomputed.Dq, - Qinv: key.Precomputed.Qinv, + switch pub := pub.(type) { + case *rsa.PublicKey: + pubBytes, _ = asn1.Marshal(rsaPublicKey{ + N: pub.N, + E: pub.E, + }) + default: + return nil, os.NewError("MarshalPKIXPublicKey: unknown public key type") } - priv.AdditionalPrimes = make([]pkcs1AdditionalRSAPrime, len(key.Precomputed.CRTValues)) - for i, values := range key.Precomputed.CRTValues { - priv.AdditionalPrimes[i].Prime = key.Primes[2+i] - priv.AdditionalPrimes[i].Exp = values.Exp - priv.AdditionalPrimes[i].Coeff = values.Coeff + pkix := pkixPublicKey{ + Algo: pkix.AlgorithmIdentifier{ + Algorithm: []int{1, 2, 840, 113549, 1, 1, 1}, + // This is a NULL parameters value which is technically + // superfluous, but most other code includes it and, by + // doing this, we match their public key hashes. + Parameters: asn1.RawValue{ + Tag: 5, + }, + }, + BitString: asn1.BitString{ + Bytes: pubBytes, + BitLength: 8 * len(pubBytes), + }, } - b, _ := asn1.Marshal(priv) - return b + ret, _ := asn1.Marshal(pkix) + return ret, nil } // These structures reflect the ASN.1 structure of X.509 certificates.: @@ -138,9 +89,9 @@ type tbsCertificate struct { Version int `asn1:"optional,explicit,default:1,tag:0"` SerialNumber *big.Int SignatureAlgorithm pkix.AlgorithmIdentifier - Issuer pkix.RDNSequence + Issuer asn1.RawValue Validity validity - Subject pkix.RDNSequence + Subject asn1.RawValue PublicKey publicKeyInfo UniqueId asn1.BitString `asn1:"optional,tag:1"` SubjectUniqueId asn1.BitString `asn1:"optional,tag:2"` @@ -339,6 +290,8 @@ type Certificate struct { Raw []byte // Complete ASN.1 DER content (certificate, signature algorithm and signature). RawTBSCertificate []byte // Certificate part of raw ASN.1 DER content. RawSubjectPublicKeyInfo []byte // DER encoded SubjectPublicKeyInfo. + RawSubject []byte // DER encoded Subject + RawIssuer []byte // DER encoded Issuer Signature []byte SignatureAlgorithm SignatureAlgorithm @@ -483,11 +436,6 @@ type basicConstraints struct { MaxPathLen int `asn1:"optional"` } -type rsaPublicKey struct { - N *big.Int - E int -} - // RFC 5280 4.2.1.4 type policyInformation struct { Policy asn1.ObjectIdentifier @@ -556,6 +504,8 @@ func parseCertificate(in *certificate) (*Certificate, os.Error) { out.Raw = in.Raw out.RawTBSCertificate = in.TBSCertificate.Raw out.RawSubjectPublicKeyInfo = in.TBSCertificate.PublicKey.Raw + out.RawSubject = in.TBSCertificate.Subject.FullBytes + out.RawIssuer = in.TBSCertificate.Issuer.FullBytes out.Signature = in.SignatureValue.RightAlign() out.SignatureAlgorithm = @@ -575,8 +525,18 @@ func parseCertificate(in *certificate) (*Certificate, os.Error) { out.Version = in.TBSCertificate.Version + 1 out.SerialNumber = in.TBSCertificate.SerialNumber - out.Issuer.FillFromRDNSequence(&in.TBSCertificate.Issuer) - out.Subject.FillFromRDNSequence(&in.TBSCertificate.Subject) + + var issuer, subject pkix.RDNSequence + if _, err := asn1.Unmarshal(in.TBSCertificate.Subject.FullBytes, &subject); err != nil { + return nil, err + } + if _, err := asn1.Unmarshal(in.TBSCertificate.Issuer.FullBytes, &issuer); err != nil { + return nil, err + } + + out.Issuer.FillFromRDNSequence(&issuer) + out.Subject.FillFromRDNSequence(&subject) + out.NotBefore = in.TBSCertificate.Validity.NotBefore out.NotAfter = in.TBSCertificate.Validity.NotAfter @@ -968,14 +928,23 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.P return } + asn1Issuer, err := asn1.Marshal(parent.Issuer.ToRDNSequence()) + if err != nil { + return + } + asn1Subject, err := asn1.Marshal(parent.Subject.ToRDNSequence()) + if err != nil { + return + } + encodedPublicKey := asn1.BitString{BitLength: len(asn1PublicKey) * 8, Bytes: asn1PublicKey} c := tbsCertificate{ Version: 2, SerialNumber: template.SerialNumber, SignatureAlgorithm: pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, - Issuer: parent.Subject.ToRDNSequence(), + Issuer: asn1.RawValue{FullBytes: asn1Issuer}, Validity: validity{template.NotBefore, template.NotAfter}, - Subject: template.Subject.ToRDNSequence(), + Subject: asn1.RawValue{FullBytes: asn1Subject}, PublicKey: publicKeyInfo{nil, pkix.AlgorithmIdentifier{Algorithm: oidRSA}, encodedPublicKey}, Extensions: extensions, } diff --git a/libgo/go/crypto/x509/x509_test.go b/libgo/go/crypto/x509/x509_test.go index dc21650..dbc5273 100644 --- a/libgo/go/crypto/x509/x509_test.go +++ b/libgo/go/crypto/x509/x509_test.go @@ -6,6 +6,7 @@ package x509 import ( "asn1" + "bytes" "big" "crypto/dsa" "crypto/rand" @@ -34,6 +35,40 @@ func TestParsePKCS1PrivateKey(t *testing.T) { } } +func TestParsePKIXPublicKey(t *testing.T) { + block, _ := pem.Decode([]byte(pemPublicKey)) + pub, err := ParsePKIXPublicKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse RSA public key: %s", err) + return + } + rsaPub, ok := pub.(*rsa.PublicKey) + if !ok { + t.Errorf("Value returned from ParsePKIXPublicKey was not an RSA public key") + return + } + + pubBytes2, err := MarshalPKIXPublicKey(rsaPub) + if err != nil { + t.Errorf("Failed to marshal RSA public key for the second time: %s", err) + return + } + if !bytes.Equal(pubBytes2, block.Bytes) { + t.Errorf("Reserialization of public key didn't match. got %x, want %x", pubBytes2, block.Bytes) + } +} + +var pemPublicKey = `-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3VoPN9PKUjKFLMwOge6+ +wnDi8sbETGIx2FKXGgqtAKpzmem53kRGEQg8WeqRmp12wgp74TGpkEXsGae7RS1k +enJCnma4fii+noGH7R0qKgHvPrI2Bwa9hzsH8tHxpyM3qrXslOmD45EH9SxIDUBJ +FehNdaPbLP1gFyahKMsdfxFJLUvbUycuZSJ2ZnIgeVxwm4qbSvZInL9Iu4FzuPtg +fINKcbbovy1qq4KvPIrXzhbY3PWDc6btxCf3SE0JdE1MCPThntB62/bLMSQ7xdDR +FF53oIpvxe/SCOymfWq/LW849Ytv3Xwod0+wzAP8STXG4HSELS4UedPYeHJJJYcZ ++QIDAQAB +-----END PUBLIC KEY----- +` + var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu diff --git a/libgo/go/csv/reader.go b/libgo/go/csv/reader.go index ea2c266..29ceeae 100644 --- a/libgo/go/csv/reader.go +++ b/libgo/go/csv/reader.go @@ -267,7 +267,7 @@ func (r *Reader) parseField() (haveField bool, delim int, err os.Error) { } if r.TrimLeadingSpace { - for unicode.IsSpace(rune) { + for rune != '\n' && unicode.IsSpace(rune) { rune, err = r.readRune() if err != nil { return false, 0, err @@ -355,7 +355,7 @@ func (r *Reader) parseField() (haveField bool, delim int, err os.Error) { c := r.column rune, err = r.readRune() if r.TrimLeadingSpace { - for unicode.IsSpace(rune) { + for rune != '\n' && unicode.IsSpace(rune) { rune, err = r.readRune() if err != nil { break diff --git a/libgo/go/csv/reader_test.go b/libgo/go/csv/reader_test.go index 0068bad..967f96b 100644 --- a/libgo/go/csv/reader_test.go +++ b/libgo/go/csv/reader_test.go @@ -127,10 +127,9 @@ field"`, Output: [][]string{{`a""b`, `c`}}, }, { - Name: "BadDoubleQuotes", - Input: `a""b,c`, - Output: [][]string{{`a""b`, `c`}}, - Error: `bare " in non-quoted-field`, Line: 1, Column: 1, + Name: "BadDoubleQuotes", + Input: `a""b,c`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 1, }, { Name: "TrimQuote", @@ -231,6 +230,23 @@ x,,, {"", "", "", ""}, }, }, + { + Name: "Issue 2366", + TrailingComma: true, + TrimLeadingSpace: true, + Input: "a,b,\nc,d,e", + Output: [][]string{ + {"a", "b", ""}, + {"c", "d", "e"}, + }, + }, + { + Name: "Issue 2366a", + TrailingComma: false, + TrimLeadingSpace: true, + Input: "a,b,\nc,d,e", + Error: "extra delimiter at end of line", + }, } func TestRead(t *testing.T) { diff --git a/libgo/go/debug/dwarf/type.go b/libgo/go/debug/dwarf/type.go index f35365e..9fa221b 100644 --- a/libgo/go/debug/dwarf/type.go +++ b/libgo/go/debug/dwarf/type.go @@ -579,6 +579,6 @@ Error: // If the parse fails, take the type out of the cache // so that the next call with this offset doesn't hit // the cache and return success. - d.typeCache[off] = nil, false + delete(d.typeCache, off) return nil, err } diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 8e55cb2..c58f736 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -17,9 +17,9 @@ import ( // A ByteOrder specifies how to convert byte sequences into // 16-, 32-, or 64-bit unsigned integers. type ByteOrder interface { - Uint16(b []byte) uint16 - Uint32(b []byte) uint32 - Uint64(b []byte) uint64 + Uint16([]byte) uint16 + Uint32([]byte) uint32 + Uint64([]byte) uint64 PutUint16([]byte, uint16) PutUint32([]byte, uint32) PutUint64([]byte, uint64) diff --git a/libgo/go/encoding/binary/varint.go b/libgo/go/encoding/binary/varint.go new file mode 100644 index 0000000..c98e0e2 --- /dev/null +++ b/libgo/go/encoding/binary/varint.go @@ -0,0 +1,149 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binary + +// This file implements "varint" encoding of 64-bit integers. +// The encoding is: +// - unsigned integers are serialized 7 bits at a time, starting with the +// least significant bits +// - the most significant bit (msb) in each output byte indicates if there +// is a continuation byte (msb = 1) +// - signed integers are mapped to unsigned integers using "zig-zag" +// encoding: Positive values x are written as 2*x + 0, negative values +// are written as 2*(^x) + 1; that is, negative numbers are complemented +// and whether to complement is encoded in bit 0. +// +// Design note: +// At most 10 bytes are needed for 64-bit values. The encoding could +// be more dense: a full 64-bit value needs an extra byte just to hold bit 63. +// Instead, the msb of the previous byte could be used to hold bit 63 since we +// know there can't be more than 64 bits. This is a trivial improvement and +// would reduce the maximum encoding length to 9 bytes. However, it breaks the +// invariant that the msb is always the "continuation bit" and thus makes the +// format incompatible with a varint encoding for larger numbers (say 128-bit). + +import ( + "io" + "os" +) + +// MaxVarintLenN is the maximum length of a varint-encoded N-bit integer. +const ( + MaxVarintLen16 = 3 + MaxVarintLen32 = 5 + MaxVarintLen64 = 10 +) + +// PutUvarint encodes a uint64 into buf and returns the number of bytes written. +func PutUvarint(buf []byte, x uint64) int { + i := 0 + for x >= 0x80 { + buf[i] = byte(x) | 0x80 + x >>= 7 + i++ + } + buf[i] = byte(x) + return i + 1 +} + +// Uvarint decodes a uint64 from buf and returns that value and the +// number of bytes read (> 0). If an error occurred, the value is 0 +// and the number of bytes n is <= 0 meaning: +// +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read +// +func Uvarint(buf []byte) (uint64, int) { + var x uint64 + var s uint + for i, b := range buf { + if b < 0x80 { + if i > 9 || i == 9 && b > 1 { + return 0, -(i + 1) // overflow + } + return x | uint64(b)<<s, i + 1 + } + x |= uint64(b&0x7f) << s + s += 7 + } + return 0, 0 +} + +// PutVarint encodes an int64 into buf and returns the number of bytes written. +func PutVarint(buf []byte, x int64) int { + ux := uint64(x) << 1 + if x < 0 { + ux = ^ux + } + return PutUvarint(buf, ux) +} + +// Varint decodes an int64 from buf and returns that value and the +// number of bytes read (> 0). If an error occurred, the value is 0 +// and the number of bytes n is <= 0 with the following meaning: +// +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read +// +func Varint(buf []byte) (int64, int) { + ux, n := Uvarint(buf) // ok to continue in presence of error + x := int64(ux >> 1) + if ux&1 != 0 { + x = ^x + } + return x, n +} + +// WriteUvarint encodes x and writes the result to w. +func WriteUvarint(w io.Writer, x uint64) os.Error { + var buf [MaxVarintLen64]byte + n := PutUvarint(buf[:], x) + _, err := w.Write(buf[0:n]) + return err +} + +var overflow = os.NewError("binary: varint overflows a 64-bit integer") + +// ReadUvarint reads an encoded unsigned integer from r and returns it as a uint64. +func ReadUvarint(r io.ByteReader) (uint64, os.Error) { + var x uint64 + var s uint + for i := 0; ; i++ { + b, err := r.ReadByte() + if err != nil { + return x, err + } + if b < 0x80 { + if i > 9 || i == 9 && b > 1 { + return x, overflow + } + return x | uint64(b)<<s, nil + } + x |= uint64(b&0x7f) << s + s += 7 + } + panic("unreachable") +} + +// WriteVarint encodes x and writes the result to w. +func WriteVarint(w io.Writer, x int64) os.Error { + ux := uint64(x) << 1 + if x < 0 { + ux = ^ux + } + return WriteUvarint(w, ux) +} + +// ReadVarint reads an encoded unsigned integer from r and returns it as a uint64. +func ReadVarint(r io.ByteReader) (int64, os.Error) { + ux, err := ReadUvarint(r) // ok to continue in presence of error + x := int64(ux >> 1) + if ux&1 != 0 { + x = ^x + } + return x, err +} diff --git a/libgo/go/encoding/binary/varint_test.go b/libgo/go/encoding/binary/varint_test.go new file mode 100644 index 0000000..ef51f09 --- /dev/null +++ b/libgo/go/encoding/binary/varint_test.go @@ -0,0 +1,182 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binary + +import ( + "bytes" + "os" + "testing" +) + +func testConstant(t *testing.T, w uint, max int) { + buf := make([]byte, MaxVarintLen64) + n := PutUvarint(buf, 1<<w-1) + if n != max { + t.Errorf("MaxVarintLen%d = %d; want %d", w, max, n) + } +} + +func TestConstants(t *testing.T) { + testConstant(t, 16, MaxVarintLen16) + testConstant(t, 32, MaxVarintLen32) + testConstant(t, 64, MaxVarintLen64) +} + +func testVarint(t *testing.T, x int64) { + buf1 := make([]byte, MaxVarintLen64) + n := PutVarint(buf1[:], x) + y, m := Varint(buf1[0:n]) + if x != y { + t.Errorf("Varint(%d): got %d", x, y) + } + if n != m { + t.Errorf("Varint(%d): got n = %d; want %d", x, m, n) + } + + var buf2 bytes.Buffer + err := WriteVarint(&buf2, x) + if err != nil { + t.Errorf("WriteVarint(%d): %s", x, err) + } + if n != buf2.Len() { + t.Errorf("WriteVarint(%d): got n = %d; want %d", x, buf2.Len(), n) + } + y, err = ReadVarint(&buf2) + if err != nil { + t.Errorf("ReadVarint(%d): %s", x, err) + } + if x != y { + t.Errorf("ReadVarint(%d): got %d", x, y) + } +} + +func testUvarint(t *testing.T, x uint64) { + buf1 := make([]byte, MaxVarintLen64) + n := PutUvarint(buf1[:], x) + y, m := Uvarint(buf1[0:n]) + if x != y { + t.Errorf("Uvarint(%d): got %d", x, y) + } + if n != m { + t.Errorf("Uvarint(%d): got n = %d; want %d", x, m, n) + } + + var buf2 bytes.Buffer + err := WriteUvarint(&buf2, x) + if err != nil { + t.Errorf("WriteUvarint(%d): %s", x, err) + } + if n != buf2.Len() { + t.Errorf("WriteUvarint(%d): got n = %d; want %d", x, buf2.Len(), n) + } + y, err = ReadUvarint(&buf2) + if err != nil { + t.Errorf("ReadUvarint(%d): %s", x, err) + } + if x != y { + t.Errorf("ReadUvarint(%d): got %d", x, y) + } +} + +var tests = []int64{ + -1 << 63, + -1<<63 + 1, + -1, + 0, + 1, + 2, + 10, + 20, + 63, + 64, + 65, + 127, + 128, + 129, + 255, + 256, + 257, + 1<<63 - 1, +} + +func TestVarint(t *testing.T) { + for _, x := range tests { + testVarint(t, x) + testVarint(t, -x) + } + for x := int64(0x7); x != 0; x <<= 1 { + testVarint(t, x) + testVarint(t, -x) + } +} + +func TestUvarint(t *testing.T) { + for _, x := range tests { + testUvarint(t, uint64(x)) + } + for x := uint64(0x7); x != 0; x <<= 1 { + testUvarint(t, x) + } +} + +func TestBufferTooSmall(t *testing.T) { + buf := []byte{0x80, 0x80, 0x80, 0x80} + for i := 0; i <= len(buf); i++ { + buf := buf[0:i] + x, n := Uvarint(buf) + if x != 0 || n != 0 { + t.Errorf("Uvarint(%v): got x = %d, n = %d", buf, x, n) + } + + x, err := ReadUvarint(bytes.NewBuffer(buf)) + if x != 0 || err != os.EOF { + t.Errorf("ReadUvarint(%v): got x = %d, err = %s", buf, x, err) + } + } +} + +func testOverflow(t *testing.T, buf []byte, n0 int, err0 os.Error) { + x, n := Uvarint(buf) + if x != 0 || n != n0 { + t.Errorf("Uvarint(%v): got x = %d, n = %d; want 0, %d", buf, x, n, n0) + } + + x, err := ReadUvarint(bytes.NewBuffer(buf)) + if x != 0 || err != err0 { + t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want 0, %s", buf, x, err, err0) + } +} + +func TestOverflow(t *testing.T) { + testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2}, -10, overflow) + testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1, 0, 0}, -13, overflow) +} + +func TestNonCanonicalZero(t *testing.T) { + buf := []byte{0x80, 0x80, 0x80, 0} + x, n := Uvarint(buf) + if x != 0 || n != 4 { + t.Errorf("Uvarint(%v): got x = %d, n = %d; want 0, 4", buf, x, n) + + } +} + +func BenchmarkPutUvarint32(b *testing.B) { + buf := make([]byte, MaxVarintLen32) + for i := 0; i < b.N; i++ { + for j := uint(0); j < MaxVarintLen32; j++ { + PutUvarint(buf, 1<<(j*7)) + } + } +} + +func BenchmarkPutUvarint64(b *testing.B) { + buf := make([]byte, MaxVarintLen64) + for i := 0; i < b.N; i++ { + for j := uint(0); j < MaxVarintLen64; j++ { + PutUvarint(buf, 1<<(j*7)) + } + } +} diff --git a/libgo/go/exec/exec.go b/libgo/go/exec/exec.go index 3b20f20..aaad508 100644 --- a/libgo/go/exec/exec.go +++ b/libgo/go/exec/exec.go @@ -63,6 +63,11 @@ type Cmd struct { Stdout io.Writer Stderr io.Writer + // ExtraFiles specifies additional open files to be inherited by the + // new process. It does not include standard input, standard output, or + // standard error. If non-nil, entry i becomes file descriptor 3+i. + ExtraFiles []*os.File + // SysProcAttr holds optional, operating system-specific attributes. // Run passes it to os.StartProcess as the os.ProcAttr's Sys field. SysProcAttr *syscall.SysProcAttr @@ -224,6 +229,7 @@ func (c *Cmd) Start() os.Error { } c.childFiles = append(c.childFiles, fd) } + c.childFiles = append(c.childFiles, c.ExtraFiles...) var err os.Error c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{ diff --git a/libgo/go/exec/exec_test.go b/libgo/go/exec/exec_test.go index 242120f..2b36e25 100644 --- a/libgo/go/exec/exec_test.go +++ b/libgo/go/exec/exec_test.go @@ -9,8 +9,10 @@ import ( "bytes" "fmt" "io" + "io/ioutil" "testing" "os" + "runtime" "strconv" "strings" ) @@ -139,6 +141,39 @@ func TestPipes(t *testing.T) { check("Wait", err) } +func TestExtraFiles(t *testing.T) { + if runtime.GOOS == "windows" { + t.Logf("no operating system support; skipping") + return + } + tf, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("TempFile: %v", err) + } + defer os.Remove(tf.Name()) + defer tf.Close() + + const text = "Hello, fd 3!" + _, err = tf.Write([]byte(text)) + if err != nil { + t.Fatalf("Write: %v", err) + } + _, err = tf.Seek(0, os.SEEK_SET) + if err != nil { + t.Fatalf("Seek: %v", err) + } + + c := helperCommand("read3") + c.ExtraFiles = []*os.File{tf} + bs, err := c.CombinedOutput() + if err != nil { + t.Fatalf("CombinedOutput: %v", err) + } + if string(bs) != text { + t.Errorf("got %q; want %q", string(bs), text) + } +} + // TestHelperProcess isn't a real test. It's used as a helper process // for TestParameterRun. func TestHelperProcess(*testing.T) { @@ -204,6 +239,14 @@ func TestHelperProcess(*testing.T) { os.Exit(1) } } + case "read3": // read fd 3 + fd3 := os.NewFile(3, "fd3") + bs, err := ioutil.ReadAll(fd3) + if err != nil { + fmt.Printf("ReadAll from fd 3: %v", err) + os.Exit(1) + } + os.Stderr.Write(bs) case "exit": n, _ := strconv.Atoi(args[0]) os.Exit(n) diff --git a/libgo/go/exec/lp_unix.go b/libgo/go/exec/lp_unix.go index 008fb11..0cd19e7 100644 --- a/libgo/go/exec/lp_unix.go +++ b/libgo/go/exec/lp_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package exec import ( diff --git a/libgo/go/ebnf/ebnf.go b/libgo/go/exp/ebnf/ebnf.go index 69da117..2ec7f00 100644 --- a/libgo/go/ebnf/ebnf.go +++ b/libgo/go/exp/ebnf/ebnf.go @@ -23,21 +23,47 @@ package ebnf import ( - "go/scanner" - "go/token" + "fmt" "os" + "scanner" "unicode" "utf8" ) // ---------------------------------------------------------------------------- +// Error handling + +type errorList []os.Error + +func (list errorList) Error() os.Error { + if len(list) == 0 { + return nil + } + return list +} + +func (list errorList) String() string { + switch len(list) { + case 0: + return "no errors" + case 1: + return list[0].String() + } + return fmt.Sprintf("%s (and %d more errors)", list[0], len(list)-1) +} + +func newError(pos scanner.Position, msg string) os.Error { + return os.NewError(fmt.Sprintf("%s: %s", pos, msg)) +} + +// ---------------------------------------------------------------------------- // Internal representation type ( // An Expression node represents a production expression. Expression interface { // Pos is the position of the first character of the syntactic construct - Pos() token.Pos + Pos() scanner.Position } // An Alternative node represents a non-empty list of alternative expressions. @@ -48,13 +74,13 @@ type ( // A Name node represents a production name. Name struct { - StringPos token.Pos + StringPos scanner.Position String string } // A Token node represents a literal. Token struct { - StringPos token.Pos + StringPos scanner.Position String string } @@ -65,50 +91,50 @@ type ( // A Group node represents a grouped expression. Group struct { - Lparen token.Pos + Lparen scanner.Position Body Expression // (body) } // An Option node represents an optional expression. Option struct { - Lbrack token.Pos + Lbrack scanner.Position Body Expression // [body] } // A Repetition node represents a repeated expression. Repetition struct { - Lbrace token.Pos + Lbrace scanner.Position Body Expression // {body} } - // A Bad node stands for pieces of source code that lead to a parse error. - Bad struct { - TokPos token.Pos - Error string // parser error message - } - // A Production node represents an EBNF production. Production struct { Name *Name Expr Expression } + // A Bad node stands for pieces of source code that lead to a parse error. + Bad struct { + TokPos scanner.Position + Error string // parser error message + } + // A Grammar is a set of EBNF productions. The map // is indexed by production name. // Grammar map[string]*Production ) -func (x Alternative) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Alternative -func (x Sequence) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Sequences -func (x *Name) Pos() token.Pos { return x.StringPos } -func (x *Token) Pos() token.Pos { return x.StringPos } -func (x *Range) Pos() token.Pos { return x.Begin.Pos() } -func (x *Group) Pos() token.Pos { return x.Lparen } -func (x *Option) Pos() token.Pos { return x.Lbrack } -func (x *Repetition) Pos() token.Pos { return x.Lbrace } -func (x *Bad) Pos() token.Pos { return x.TokPos } -func (x *Production) Pos() token.Pos { return x.Name.Pos() } +func (x Alternative) Pos() scanner.Position { return x[0].Pos() } // the parser always generates non-empty Alternative +func (x Sequence) Pos() scanner.Position { return x[0].Pos() } // the parser always generates non-empty Sequences +func (x *Name) Pos() scanner.Position { return x.StringPos } +func (x *Token) Pos() scanner.Position { return x.StringPos } +func (x *Range) Pos() scanner.Position { return x.Begin.Pos() } +func (x *Group) Pos() scanner.Position { return x.Lparen } +func (x *Option) Pos() scanner.Position { return x.Lbrack } +func (x *Repetition) Pos() scanner.Position { return x.Lbrace } +func (x *Production) Pos() scanner.Position { return x.Name.Pos() } +func (x *Bad) Pos() scanner.Position { return x.TokPos } // ---------------------------------------------------------------------------- // Grammar verification @@ -119,15 +145,14 @@ func isLexical(name string) bool { } type verifier struct { - fset *token.FileSet - scanner.ErrorVector + errors errorList worklist []*Production reached Grammar // set of productions reached from (and including) the root production grammar Grammar } -func (v *verifier) error(pos token.Pos, msg string) { - v.Error(v.fset.Position(pos), msg) +func (v *verifier) error(pos scanner.Position, msg string) { + v.errors = append(v.errors, newError(pos, msg)) } func (v *verifier) push(prod *Production) { @@ -187,24 +212,23 @@ func (v *verifier) verifyExpr(expr Expression, lexical bool) { v.verifyExpr(x.Body, lexical) case *Repetition: v.verifyExpr(x.Body, lexical) + case *Bad: + v.error(x.Pos(), x.Error) default: - panic("unreachable") + panic(fmt.Sprintf("internal error: unexpected type %T", expr)) } } -func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) { +func (v *verifier) verify(grammar Grammar, start string) { // find root production root, found := grammar[start] if !found { - // token.NoPos doesn't require a file set; - // ok to set v.fset only afterwards - v.error(token.NoPos, "no start production "+start) + var noPos scanner.Position + v.error(noPos, "no start production "+start) return } // initialize verifier - v.fset = fset - v.ErrorVector.Reset() v.worklist = v.worklist[0:0] v.reached = make(Grammar) v.grammar = grammar @@ -238,8 +262,8 @@ func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) { // // Position information is interpreted relative to the file set fset. // -func Verify(fset *token.FileSet, grammar Grammar, start string) os.Error { +func Verify(grammar Grammar, start string) os.Error { var v verifier - v.verify(fset, grammar, start) - return v.GetError(scanner.Sorted) + v.verify(grammar, start) + return v.errors.Error() } diff --git a/libgo/go/ebnf/ebnf_test.go b/libgo/go/exp/ebnf/ebnf_test.go index b086fac..8cfd6b9 100644 --- a/libgo/go/ebnf/ebnf_test.go +++ b/libgo/go/exp/ebnf/ebnf_test.go @@ -5,13 +5,10 @@ package ebnf import ( - "go/token" - "io/ioutil" + "bytes" "testing" ) -var fset = token.NewFileSet() - var goodGrammars = []string{ `Program = .`, @@ -46,18 +43,19 @@ var badGrammars = []string{ `Program = {} .`, } -func checkGood(t *testing.T, filename string, src []byte) { - grammar, err := Parse(fset, filename, src) +func checkGood(t *testing.T, src string) { + grammar, err := Parse("", bytes.NewBuffer([]byte(src))) if err != nil { t.Errorf("Parse(%s) failed: %v", src, err) + return } - if err = Verify(fset, grammar, "Program"); err != nil { + if err = Verify(grammar, "Program"); err != nil { t.Errorf("Verify(%s) failed: %v", src, err) } } -func checkBad(t *testing.T, filename string, src []byte) { - _, err := Parse(fset, filename, src) +func checkBad(t *testing.T, src string) { + _, err := Parse("", bytes.NewBuffer([]byte(src))) if err == nil { t.Errorf("Parse(%s) should have failed", src) } @@ -65,23 +63,9 @@ func checkBad(t *testing.T, filename string, src []byte) { func TestGrammars(t *testing.T) { for _, src := range goodGrammars { - checkGood(t, "", []byte(src)) + checkGood(t, src) } for _, src := range badGrammars { - checkBad(t, "", []byte(src)) - } -} - -var files = []string{ -// TODO(gri) add some test files -} - -func TestFiles(t *testing.T) { - for _, filename := range files { - src, err := ioutil.ReadFile(filename) - if err != nil { - t.Fatal(err) - } - checkGood(t, filename, src) + checkBad(t, src) } } diff --git a/libgo/go/ebnf/parser.go b/libgo/go/exp/ebnf/parser.go index ef2fac0..2dbbefb 100644 --- a/libgo/go/ebnf/parser.go +++ b/libgo/go/exp/ebnf/parser.go @@ -5,51 +5,47 @@ package ebnf import ( - "go/scanner" - "go/token" + "io" "os" + "scanner" "strconv" ) type parser struct { - fset *token.FileSet - scanner.ErrorVector + errors errorList scanner scanner.Scanner - pos token.Pos // token position - tok token.Token // one token look-ahead - lit string // token literal + pos scanner.Position // token position + tok int // one token look-ahead + lit string // token literal } func (p *parser) next() { - p.pos, p.tok, p.lit = p.scanner.Scan() - if p.tok.IsKeyword() { - // TODO Should keyword mapping always happen outside scanner? - // Or should there be a flag to scanner to enable keyword mapping? - p.tok = token.IDENT - } + p.tok = p.scanner.Scan() + p.pos = p.scanner.Position + p.lit = p.scanner.TokenText() } -func (p *parser) error(pos token.Pos, msg string) { - p.Error(p.fset.Position(pos), msg) +func (p *parser) error(pos scanner.Position, msg string) { + p.errors = append(p.errors, newError(pos, msg)) } -func (p *parser) errorExpected(pos token.Pos, msg string) { - msg = "expected " + msg - if pos == p.pos { +func (p *parser) errorExpected(pos scanner.Position, msg string) { + msg = `expected "` + msg + `"` + if pos.Offset == p.pos.Offset { // the error happened at the current position; // make the error message more specific - msg += ", found '" + p.tok.String() + "'" - if p.tok.IsLiteral() { + msg += ", found " + scanner.TokenString(p.tok) + if p.tok < 0 { msg += " " + p.lit } } p.error(pos, msg) } -func (p *parser) expect(tok token.Token) token.Pos { +func (p *parser) expect(tok int) scanner.Position { pos := p.pos if p.tok != tok { - p.errorExpected(pos, "'"+tok.String()+"'") + p.errorExpected(pos, scanner.TokenString(tok)) } p.next() // make progress in any case return pos @@ -58,21 +54,21 @@ func (p *parser) expect(tok token.Token) token.Pos { func (p *parser) parseIdentifier() *Name { pos := p.pos name := p.lit - p.expect(token.IDENT) + p.expect(scanner.Ident) return &Name{pos, name} } func (p *parser) parseToken() *Token { pos := p.pos value := "" - if p.tok == token.STRING { + if p.tok == scanner.String { value, _ = strconv.Unquote(p.lit) // Unquote may fail with an error, but only if the scanner found // an illegal string in the first place. In this case the error // has already been reported. p.next() } else { - p.expect(token.STRING) + p.expect(scanner.String) } return &Token{pos, value} } @@ -82,32 +78,32 @@ func (p *parser) parseTerm() (x Expression) { pos := p.pos switch p.tok { - case token.IDENT: + case scanner.Ident: x = p.parseIdentifier() - case token.STRING: + case scanner.String: tok := p.parseToken() x = tok - const ellipsis = "…" // U+2026, the horizontal ellipsis character - if p.tok == token.ILLEGAL && p.lit == ellipsis { + const ellipsis = '…' // U+2026, the horizontal ellipsis character + if p.tok == ellipsis { p.next() x = &Range{tok, p.parseToken()} } - case token.LPAREN: + case '(': p.next() x = &Group{pos, p.parseExpression()} - p.expect(token.RPAREN) + p.expect(')') - case token.LBRACK: + case '[': p.next() x = &Option{pos, p.parseExpression()} - p.expect(token.RBRACK) + p.expect(']') - case token.LBRACE: + case '{': p.next() x = &Repetition{pos, p.parseExpression()} - p.expect(token.RBRACE) + p.expect('}') } return x @@ -137,7 +133,7 @@ func (p *parser) parseExpression() Expression { for { list = append(list, p.parseSequence()) - if p.tok != token.OR { + if p.tok != '|' { break } p.next() @@ -154,24 +150,22 @@ func (p *parser) parseExpression() Expression { func (p *parser) parseProduction() *Production { name := p.parseIdentifier() - p.expect(token.ASSIGN) + p.expect('=') var expr Expression - if p.tok != token.PERIOD { + if p.tok != '.' { expr = p.parseExpression() } - p.expect(token.PERIOD) + p.expect('.') return &Production{name, expr} } -func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar { - // initialize parser - p.fset = fset - p.ErrorVector.Reset() - p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, scanner.AllowIllegalChars) +func (p *parser) parse(filename string, src io.Reader) Grammar { + p.scanner.Init(src) + p.scanner.Filename = filename p.next() // initializes pos, tok, lit grammar := make(Grammar) - for p.tok != token.EOF { + for p.tok != scanner.EOF { prod := p.parseProduction() name := prod.Name.String if _, found := grammar[name]; !found { @@ -187,11 +181,11 @@ func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar // Parse parses a set of EBNF productions from source src. // It returns a set of productions. Errors are reported // for incorrect syntax and if a production is declared -// more than once. Position information is recorded relative -// to the file set fset. +// more than once; the filename is used only for error +// positions. // -func Parse(fset *token.FileSet, filename string, src []byte) (Grammar, os.Error) { +func Parse(filename string, src io.Reader) (Grammar, os.Error) { var p parser - grammar := p.parse(fset, filename, src) - return grammar, p.GetError(scanner.Sorted) + grammar := p.parse(filename, src) + return grammar, p.errors.Error() } diff --git a/libgo/go/exp/ebnflint/doc.go b/libgo/go/exp/ebnflint/doc.go new file mode 100644 index 0000000..f35976e --- /dev/null +++ b/libgo/go/exp/ebnflint/doc.go @@ -0,0 +1,22 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Ebnflint verifies that EBNF productions are consistent and gramatically correct. +It reads them from an HTML document such as the Go specification. + +Grammar productions are grouped in boxes demarcated by the HTML elements + <pre class="ebnf"> + </pre> + + +Usage: + ebnflint [--start production] [file] + +The --start flag specifies the name of the start production for +the grammar; it defaults to "Start". + +*/ +package documentation diff --git a/libgo/go/exp/ebnflint/ebnflint.go b/libgo/go/exp/ebnflint/ebnflint.go new file mode 100644 index 0000000..c827716 --- /dev/null +++ b/libgo/go/exp/ebnflint/ebnflint.go @@ -0,0 +1,109 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "exp/ebnf" + "flag" + "fmt" + "go/scanner" + "go/token" + "io/ioutil" + "os" + "path/filepath" +) + +var fset = token.NewFileSet() +var start = flag.String("start", "Start", "name of start production") + +func usage() { + fmt.Fprintf(os.Stderr, "usage: ebnflint [flags] [filename]\n") + flag.PrintDefaults() + os.Exit(1) +} + +// Markers around EBNF sections in .html files +var ( + open = []byte(`<pre class="ebnf">`) + close = []byte(`</pre>`) +) + +func report(err os.Error) { + scanner.PrintError(os.Stderr, err) + os.Exit(1) +} + +func extractEBNF(src []byte) []byte { + var buf bytes.Buffer + + for { + // i = beginning of EBNF text + i := bytes.Index(src, open) + if i < 0 { + break // no EBNF found - we are done + } + i += len(open) + + // write as many newlines as found in the excluded text + // to maintain correct line numbers in error messages + for _, ch := range src[0:i] { + if ch == '\n' { + buf.WriteByte('\n') + } + } + + // j = end of EBNF text (or end of source) + j := bytes.Index(src[i:], close) // close marker + if j < 0 { + j = len(src) - i + } + j += i + + // copy EBNF text + buf.Write(src[i:j]) + + // advance + src = src[j:] + } + + return buf.Bytes() +} + +func main() { + flag.Parse() + + var ( + filename string + src []byte + err os.Error + ) + switch flag.NArg() { + case 0: + filename = "<stdin>" + src, err = ioutil.ReadAll(os.Stdin) + case 1: + filename = flag.Arg(0) + src, err = ioutil.ReadFile(filename) + default: + usage() + } + if err != nil { + report(err) + } + + if filepath.Ext(filename) == ".html" || bytes.Index(src, open) >= 0 { + src = extractEBNF(src) + } + + grammar, err := ebnf.Parse(filename, bytes.NewBuffer(src)) + if err != nil { + report(err) + } + + if err = ebnf.Verify(grammar, *start); err != nil { + report(err) + } +} diff --git a/libgo/go/exp/gotype/doc.go b/libgo/go/exp/gotype/doc.go new file mode 100644 index 0000000..1aa0faa --- /dev/null +++ b/libgo/go/exp/gotype/doc.go @@ -0,0 +1,61 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +The gotype command does syntactic and semantic analysis of Go files +and packages similar to the analysis performed by the front-end of +a Go compiler. Errors are reported if the analysis fails; otherwise +gotype is quiet (unless -v is set). + +Without a list of paths, gotype processes the standard input, which must +be the source of a single package file. + +Given a list of file names, each file must be a source file belonging to +the same package unless the package name is explicitly specified with the +-p flag. + +Given a directory name, gotype collects all .go files in the directory +and processes them as if they were provided as an explicit list of file +names. Each directory is processed independently. Files starting with . +or not ending in .go are ignored. + +Usage: + gotype [flags] [path ...] + +The flags are: + -e + Print all (including spurious) errors. + -p pkgName + Process only those files in package pkgName. + -r + Recursively process subdirectories. + -v + Verbose mode. + +Debugging flags: + -ast + Print AST (disables concurrent parsing). + -trace + Print parse trace (disables concurrent parsing). + + +Examples + +To check the files file.go, old.saved, and .ignored: + + gotype file.go old.saved .ignored + +To check all .go files belonging to package main in the current directory +and recursively in all subdirectories: + + gotype -p main -r . + +To verify the output of a pipe: + + echo "package foo" | gotype + +*/ +package documentation + +// BUG(gri): At the moment, only single-file scope analysis is performed. diff --git a/libgo/go/exp/gotype/gotype.go b/libgo/go/exp/gotype/gotype.go new file mode 100644 index 0000000..9199213 --- /dev/null +++ b/libgo/go/exp/gotype/gotype.go @@ -0,0 +1,192 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "exp/types" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/scanner" + "go/token" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +var ( + // main operation modes + pkgName = flag.String("p", "", "process only those files in package pkgName") + recursive = flag.Bool("r", false, "recursively process subdirectories") + verbose = flag.Bool("v", false, "verbose mode") + allErrors = flag.Bool("e", false, "print all (including spurious) errors") + + // debugging support + printTrace = flag.Bool("trace", false, "print parse trace") + printAST = flag.Bool("ast", false, "print AST") +) + +var exitCode = 0 + +func usage() { + fmt.Fprintf(os.Stderr, "usage: gotype [flags] [path ...]\n") + flag.PrintDefaults() + os.Exit(2) +} + +func report(err os.Error) { + scanner.PrintError(os.Stderr, err) + exitCode = 2 +} + +// parse returns the AST for the Go source src. +// The filename is for error reporting only. +// The result is nil if there were errors or if +// the file does not belong to the -p package. +func parse(fset *token.FileSet, filename string, src []byte) *ast.File { + if *verbose { + fmt.Println(filename) + } + + // ignore files with different package name + if *pkgName != "" { + file, err := parser.ParseFile(fset, filename, src, parser.PackageClauseOnly) + if err != nil { + report(err) + return nil + } + if file.Name.Name != *pkgName { + if *verbose { + fmt.Printf("\tignored (package %s)\n", file.Name.Name) + } + return nil + } + } + + // parse entire file + mode := parser.DeclarationErrors + if *allErrors { + mode |= parser.SpuriousErrors + } + if *printTrace { + mode |= parser.Trace + } + file, err := parser.ParseFile(fset, filename, src, mode) + if err != nil { + report(err) + return nil + } + if *printAST { + ast.Print(fset, file) + } + + return file +} + +func parseStdin(fset *token.FileSet) (files map[string]*ast.File) { + files = make(map[string]*ast.File) + src, err := ioutil.ReadAll(os.Stdin) + if err != nil { + report(err) + return + } + const filename = "<standard input>" + if file := parse(fset, filename, src); file != nil { + files[filename] = file + } + return +} + +func parseFiles(fset *token.FileSet, filenames []string) (files map[string]*ast.File) { + files = make(map[string]*ast.File) + for _, filename := range filenames { + src, err := ioutil.ReadFile(filename) + if err != nil { + report(err) + continue + } + if file := parse(fset, filename, src); file != nil { + if files[filename] != nil { + report(os.NewError(fmt.Sprintf("%q: duplicate file", filename))) + continue + } + files[filename] = file + } + } + return +} + +func isGoFilename(filename string) bool { + // ignore non-Go files + return !strings.HasPrefix(filename, ".") && strings.HasSuffix(filename, ".go") +} + +func processDirectory(dirname string) { + f, err := os.Open(dirname) + if err != nil { + report(err) + return + } + filenames, err := f.Readdirnames(-1) + f.Close() + if err != nil { + report(err) + // continue since filenames may not be empty + } + for i, filename := range filenames { + filenames[i] = filepath.Join(dirname, filename) + } + processFiles(filenames, false) +} + +func processFiles(filenames []string, allFiles bool) { + i := 0 + for _, filename := range filenames { + switch info, err := os.Stat(filename); { + case err != nil: + report(err) + case info.IsRegular(): + if allFiles || isGoFilename(info.Name) { + filenames[i] = filename + i++ + } + case info.IsDirectory(): + if allFiles || *recursive { + processDirectory(filename) + } + } + } + fset := token.NewFileSet() + processPackage(fset, parseFiles(fset, filenames[0:i])) +} + +func processPackage(fset *token.FileSet, files map[string]*ast.File) { + // make a package (resolve all identifiers) + pkg, err := ast.NewPackage(fset, files, types.GcImporter, types.Universe) + if err != nil { + report(err) + return + } + _, err = types.Check(fset, pkg) + if err != nil { + report(err) + } +} + +func main() { + flag.Usage = usage + flag.Parse() + + if flag.NArg() == 0 { + fset := token.NewFileSet() + processPackage(fset, parseStdin(fset)) + } else { + processFiles(flag.Args(), true) + } + + os.Exit(exitCode) +} diff --git a/libgo/go/exp/gotype/gotype_test.go b/libgo/go/exp/gotype/gotype_test.go new file mode 100644 index 0000000..8732d4c --- /dev/null +++ b/libgo/go/exp/gotype/gotype_test.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "path/filepath" + "runtime" + "testing" +) + +func runTest(t *testing.T, path, pkg string) { + exitCode = 0 + *pkgName = pkg + *recursive = false + + if pkg == "" { + processFiles([]string{path}, true) + } else { + processDirectory(path) + } + + if exitCode != 0 { + t.Errorf("processing %s failed: exitCode = %d", path, exitCode) + } +} + +var tests = []struct { + path string + pkg string +}{ + // individual files + {"testdata/test1.go", ""}, + + // directories + {filepath.Join(runtime.GOROOT(), "src/pkg/go/ast"), "ast"}, + {filepath.Join(runtime.GOROOT(), "src/pkg/go/doc"), "doc"}, + {filepath.Join(runtime.GOROOT(), "src/pkg/go/token"), "scanner"}, + {filepath.Join(runtime.GOROOT(), "src/pkg/go/scanner"), "scanner"}, + {filepath.Join(runtime.GOROOT(), "src/pkg/go/parser"), "parser"}, + {filepath.Join(runtime.GOROOT(), "src/pkg/exp/types"), "types"}, +} + +func Test(t *testing.T) { + for _, test := range tests { + runTest(t, test.path, test.pkg) + } +} diff --git a/libgo/go/exp/gotype/testdata/test1.go b/libgo/go/exp/gotype/testdata/test1.go new file mode 100644 index 0000000..a3298e6 --- /dev/null +++ b/libgo/go/exp/gotype/testdata/test1.go @@ -0,0 +1,23 @@ +package p + +func _() { + // the scope of a local type declaration starts immediately after the type name + type T struct{ _ *T } +} + +func _(x interface{}) { + // the variable defined by a TypeSwitchGuard is declared in each TypeCaseClause + switch t := x.(type) { + case int: + _ = t + case float32: + _ = t + default: + _ = t + } + + // the variable defined by a TypeSwitchGuard must not conflict with other + // variables declared in the initial simple statement + switch t := 0; t := x.(type) { + } +} diff --git a/libgo/go/exp/gui/x11/conn.go b/libgo/go/exp/gui/x11/conn.go index 98c65b9..bf94bca 100644 --- a/libgo/go/exp/gui/x11/conn.go +++ b/libgo/go/exp/gui/x11/conn.go @@ -621,7 +621,7 @@ func NewWindowDisplay(display string) (gui.Window, os.Error) { return nil, err } - c.img = image.NewRGBA(windowWidth, windowHeight) + c.img = image.NewRGBA(image.Rect(0, 0, windowWidth, windowHeight)) c.eventc = make(chan interface{}, 16) c.flush = make(chan bool, 1) go c.readSocket() diff --git a/libgo/go/exp/norm/composition.go b/libgo/go/exp/norm/composition.go index b2d2aba..1d72223 100644 --- a/libgo/go/exp/norm/composition.go +++ b/libgo/go/exp/norm/composition.go @@ -7,27 +7,46 @@ package norm import "utf8" const ( - maxCombiningChars = 30 + 2 // +2 to hold CGJ and Hangul overflow. + maxCombiningChars = 30 + maxBufferSize = maxCombiningChars + 2 // +1 to hold starter +1 to hold CGJ maxBackRunes = maxCombiningChars - 1 maxNFCExpansion = 3 // NFC(0x1D160) maxNFKCExpansion = 18 // NFKC(0xFDFA) - maxRuneSizeInDecomp = 4 - // Need to multiply by 2 as we don't reuse byte buffer space for recombining. - maxByteBufferSize = 2 * maxRuneSizeInDecomp * maxCombiningChars // 256 + maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128 ) // reorderBuffer is used to normalize a single segment. Characters inserted with -// insert() are decomposed and reordered based on CCC. The compose() method can +// insert are decomposed and reordered based on CCC. The compose method can // be used to recombine characters. Note that the byte buffer does not hold // the UTF-8 characters in order. Only the rune array is maintained in sorted -// order. flush() writes the resulting segment to a byte array. +// order. flush writes the resulting segment to a byte array. type reorderBuffer struct { - rune [maxCombiningChars]runeInfo // Per character info. - byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. - nrune int // Number of runeInfos. - nbyte uint8 // Number or bytes. + rune [maxBufferSize]runeInfo // Per character info. + byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. + nrune int // Number of runeInfos. + nbyte uint8 // Number or bytes. f formInfo + + src input + nsrc int + srcBytes inputBytes + srcString inputString + tmpBytes inputBytes +} + +func (rb *reorderBuffer) init(f Form, src []byte) { + rb.f = *formTable[f] + rb.srcBytes = inputBytes(src) + rb.src = &rb.srcBytes + rb.nsrc = len(src) +} + +func (rb *reorderBuffer) initString(f Form, src string) { + rb.f = *formTable[f] + rb.srcString = inputString(src) + rb.src = &rb.srcString + rb.nsrc = len(src) } // reset discards all characters from the buffer. @@ -49,10 +68,10 @@ func (rb *reorderBuffer) flush(out []byte) []byte { // insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. // It returns false if the buffer is not large enough to hold the rune. -// It is used internally by insert. +// It is used internally by insert and insertString only. func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { n := rb.nrune - if n >= maxCombiningChars { + if n >= maxCombiningChars+1 { return false } b := rb.rune[:] @@ -68,7 +87,7 @@ func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { } rb.nrune += 1 pos := uint8(rb.nbyte) - rb.nbyte += info.size + rb.nbyte += utf8.UTFMax info.pos = pos b[n] = info return true @@ -76,53 +95,32 @@ func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { // insert inserts the given rune in the buffer ordered by CCC. // It returns true if the buffer was large enough to hold the decomposed rune. -func (rb *reorderBuffer) insert(src []byte, info runeInfo) bool { - if info.size == 3 && isHangul(src) { - rune, _ := utf8.DecodeRune(src) - return rb.decomposeHangul(uint32(rune)) - } - pos := rb.nbyte - if info.flags.hasDecomposition() { - dcomp := rb.f.decompose(src) - for i := 0; i < len(dcomp); i += int(info.size) { - info = rb.f.info(dcomp[i:]) - if !rb.insertOrdered(info) { - return false - } - } - copy(rb.byte[pos:], dcomp) - } else { - if !rb.insertOrdered(info) { - return false +func (rb *reorderBuffer) insert(src input, i int, info runeInfo) bool { + if info.size == 3 { + if rune := src.hangul(i); rune != 0 { + return rb.decomposeHangul(uint32(rune)) } - copy(rb.byte[pos:], src[:info.size]) } - return true -} - -// insertString inserts the given rune in the buffer ordered by CCC. -// It returns true if the buffer was large enough to hold the decomposed rune. -func (rb *reorderBuffer) insertString(src string, info runeInfo) bool { - if info.size == 3 && isHangulString(src) { - rune, _ := utf8.DecodeRuneInString(src) - return rb.decomposeHangul(uint32(rune)) - } - pos := rb.nbyte - dcomp := rb.f.decomposeString(src) - dn := len(dcomp) - if dn != 0 { - for i := 0; i < dn; i += int(info.size) { - info = rb.f.info(dcomp[i:]) + if info.flags.hasDecomposition() { + dcomp := rb.f.decompose(src, i) + rb.tmpBytes = inputBytes(dcomp) + for i := 0; i < len(dcomp); { + info = rb.f.info(&rb.tmpBytes, i) + pos := rb.nbyte if !rb.insertOrdered(info) { return false } + end := i + int(info.size) + copy(rb.byte[pos:], dcomp[i:end]) + i = end } - copy(rb.byte[pos:], dcomp) } else { + // insertOrder changes nbyte + pos := rb.nbyte if !rb.insertOrdered(info) { return false } - copy(rb.byte[pos:], src[:info.size]) + src.copySlice(rb.byte[pos:], i, i+int(info.size)) } return true } @@ -131,17 +129,16 @@ func (rb *reorderBuffer) insertString(src string, info runeInfo) bool { func (rb *reorderBuffer) appendRune(rune uint32) { bn := rb.nbyte sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) - rb.nbyte += uint8(sz) + rb.nbyte += utf8.UTFMax rb.rune[rb.nrune] = runeInfo{bn, uint8(sz), 0, 0} rb.nrune++ } // assignRune sets a rune at position pos. It is used for Hangul and recomposition. func (rb *reorderBuffer) assignRune(pos int, rune uint32) { - bn := rb.nbyte + bn := rb.rune[pos].pos sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) rb.rune[pos] = runeInfo{bn, uint8(sz), 0, 0} - rb.nbyte += uint8(sz) } // runeAt returns the rune at position n. It is used for Hangul and recomposition. @@ -259,11 +256,10 @@ func (rb *reorderBuffer) decomposeHangul(rune uint32) bool { // combineHangul algorithmically combines Jamo character components into Hangul. // See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul. -func (rb *reorderBuffer) combineHangul() { - k := 1 +func (rb *reorderBuffer) combineHangul(s, i, k int) { b := rb.rune[:] bn := rb.nrune - for s, i := 0, 1; i < bn; i++ { + for ; i < bn; i++ { cccB := b[k-1].ccc cccC := b[i].ccc if cccB == 0 { @@ -305,14 +301,17 @@ func (rb *reorderBuffer) compose() { // blocked from S if and only if there is some character B between S // and C, and either B is a starter or it has the same or higher // combining class as C." + bn := rb.nrune + if bn == 0 { + return + } k := 1 b := rb.rune[:] - bn := rb.nrune for s, i := 0, 1; i < bn; i++ { if isJamoVT(rb.bytesAt(i)) { // Redo from start in Hangul mode. Necessary to support // U+320E..U+321E in NFKC mode. - rb.combineHangul() + rb.combineHangul(s, i, k) return } ii := b[i] diff --git a/libgo/go/exp/norm/composition_test.go b/libgo/go/exp/norm/composition_test.go index 195a0c1..ce9caaf 100644 --- a/libgo/go/exp/norm/composition_test.go +++ b/libgo/go/exp/norm/composition_test.go @@ -15,21 +15,19 @@ type TestCase struct { type insertFunc func(rb *reorderBuffer, rune int) bool func insert(rb *reorderBuffer, rune int) bool { - b := []byte(string(rune)) - return rb.insert(b, rb.f.info(b)) + src := inputString(string(rune)) + return rb.insert(src, 0, rb.f.info(src, 0)) } -func insertString(rb *reorderBuffer, rune int) bool { - s := string(rune) - return rb.insertString(s, rb.f.infoString(s)) -} - -func runTests(t *testing.T, name string, rb *reorderBuffer, f insertFunc, tests []TestCase) { +func runTests(t *testing.T, name string, fm Form, f insertFunc, tests []TestCase) { + rb := reorderBuffer{} + rb.init(fm, nil) for i, test := range tests { rb.reset() for j, rune := range test.in { b := []byte(string(rune)) - if !rb.insert(b, rb.f.info(b)) { + src := inputBytes(b) + if !rb.insert(src, 0, rb.f.info(src, 0)) { t.Errorf("%s:%d: insert failed for rune %d", name, i, j) } } @@ -50,7 +48,8 @@ func runTests(t *testing.T, name string, rb *reorderBuffer, f insertFunc, tests } func TestFlush(t *testing.T) { - rb := &reorderBuffer{f: *formTable[NFC]} + rb := reorderBuffer{} + rb.init(NFC, nil) out := make([]byte, 0) out = rb.flush(out) @@ -59,7 +58,7 @@ func TestFlush(t *testing.T) { } for _, r := range []int("world!") { - insert(rb, r) + insert(&rb, r) } out = []byte("Hello ") @@ -88,13 +87,7 @@ var insertTests = []TestCase{ } func TestInsert(t *testing.T) { - rb := &reorderBuffer{f: *formTable[NFD]} - runTests(t, "TestInsert", rb, insert, insertTests) -} - -func TestInsertString(t *testing.T) { - rb := &reorderBuffer{f: *formTable[NFD]} - runTests(t, "TestInsertString", rb, insertString, insertTests) + runTests(t, "TestInsert", NFD, insert, insertTests) } var decompositionNFDTest = []TestCase{ @@ -113,11 +106,8 @@ var decompositionNFKDTest = []TestCase{ } func TestDecomposition(t *testing.T) { - rb := &reorderBuffer{} - rb.f = *formTable[NFD] - runTests(t, "TestDecompositionNFD", rb, insert, decompositionNFDTest) - rb.f = *formTable[NFKD] - runTests(t, "TestDecompositionNFKD", rb, insert, decompositionNFKDTest) + runTests(t, "TestDecompositionNFD", NFD, insert, decompositionNFDTest) + runTests(t, "TestDecompositionNFKD", NFKD, insert, decompositionNFKDTest) } var compositionTest = []TestCase{ @@ -133,6 +123,5 @@ var compositionTest = []TestCase{ } func TestComposition(t *testing.T) { - rb := &reorderBuffer{f: *formTable[NFC]} - runTests(t, "TestComposition", rb, insert, compositionTest) + runTests(t, "TestComposition", NFC, insert, compositionTest) } diff --git a/libgo/go/exp/norm/forminfo.go b/libgo/go/exp/norm/forminfo.go index ee3edb8..d06a006 100644 --- a/libgo/go/exp/norm/forminfo.go +++ b/libgo/go/exp/norm/forminfo.go @@ -15,10 +15,8 @@ type runeInfo struct { // functions dispatchable per form type boundaryFunc func(f *formInfo, info runeInfo) bool -type lookupFunc func(b []byte) runeInfo -type lookupFuncString func(s string) runeInfo -type decompFunc func(b []byte) []byte -type decompFuncString func(s string) []byte +type lookupFunc func(b input, i int) runeInfo +type decompFunc func(b input, i int) []byte // formInfo holds Form-specific functions and tables. type formInfo struct { @@ -26,12 +24,10 @@ type formInfo struct { composing, compatibility bool // form type - decompose decompFunc - decomposeString decompFuncString - info lookupFunc - infoString lookupFuncString - boundaryBefore boundaryFunc - boundaryAfter boundaryFunc + decompose decompFunc + info lookupFunc + boundaryBefore boundaryFunc + boundaryAfter boundaryFunc } var formTable []*formInfo @@ -46,14 +42,10 @@ func init() { if Form(i) == NFKD || Form(i) == NFKC { f.compatibility = true f.decompose = decomposeNFKC - f.decomposeString = decomposeStringNFKC f.info = lookupInfoNFKC - f.infoString = lookupInfoStringNFKC } else { f.decompose = decomposeNFC - f.decomposeString = decomposeStringNFC f.info = lookupInfoNFC - f.infoString = lookupInfoStringNFC } if Form(i) == NFC || Form(i) == NFKC { f.composing = true @@ -77,7 +69,7 @@ func decompBoundary(f *formInfo, info runeInfo) bool { } func compBoundaryBefore(f *formInfo, info runeInfo) bool { - if info.ccc == 0 && info.flags.isYesC() { + if info.ccc == 0 && !info.flags.combinesBackward() { return true } // We assume that the CCC of the first character in a decomposition @@ -89,9 +81,7 @@ func compBoundaryBefore(f *formInfo, info runeInfo) bool { func compBoundaryAfter(f *formInfo, info runeInfo) bool { // This misses values where the last char in a decomposition is a // boundary such as Hangul with JamoT. - // TODO(mpvl): verify this does not lead to segments that do - // not fit in the reorderBuffer. - return info.flags.isInert() + return info.isInert() } // We pack quick check data in 4 bits: @@ -110,41 +100,30 @@ func (i qcInfo) isNoC() bool { return i&0x6 == 0x2 } func (i qcInfo) isMaybe() bool { return i&0x4 != 0 } func (i qcInfo) isYesD() bool { return i&0x1 == 0 } func (i qcInfo) isNoD() bool { return i&0x1 != 0 } -func (i qcInfo) isInert() bool { return i&0xf == 0 } func (i qcInfo) combinesForward() bool { return i&0x8 != 0 } func (i qcInfo) combinesBackward() bool { return i&0x4 != 0 } // == isMaybe func (i qcInfo) hasDecomposition() bool { return i&0x1 != 0 } // == isNoD +func (r runeInfo) isInert() bool { + return r.flags&0xf == 0 && r.ccc == 0 +} + // Wrappers for tables.go // The 16-bit value of the decompostion tries is an index into a byte // array of UTF-8 decomposition sequences. The first byte is the number // of bytes in the decomposition (excluding this length byte). The actual // sequence starts at the offset+1. -func decomposeNFC(b []byte) []byte { - p := nfcDecompTrie.lookupUnsafe(b) - n := decomps[p] - p++ - return decomps[p : p+uint16(n)] -} - -func decomposeNFKC(b []byte) []byte { - p := nfkcDecompTrie.lookupUnsafe(b) - n := decomps[p] - p++ - return decomps[p : p+uint16(n)] -} - -func decomposeStringNFC(s string) []byte { - p := nfcDecompTrie.lookupStringUnsafe(s) +func decomposeNFC(s input, i int) []byte { + p := s.decomposeNFC(i) n := decomps[p] p++ return decomps[p : p+uint16(n)] } -func decomposeStringNFKC(s string) []byte { - p := nfkcDecompTrie.lookupStringUnsafe(s) +func decomposeNFKC(s input, i int) []byte { + p := s.decomposeNFKC(i) n := decomps[p] p++ return decomps[p : p+uint16(n)] @@ -167,22 +146,12 @@ func combine(a, b uint32) uint32 { // 0..7 CCC value. // 8..11 qcInfo for NFC/NFD // 12..15 qcInfo for NFKC/NFKD -func lookupInfoNFC(b []byte) runeInfo { - v, sz := charInfoTrie.lookup(b) +func lookupInfoNFC(b input, i int) runeInfo { + v, sz := b.charinfo(i) return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 8)} } -func lookupInfoStringNFC(s string) runeInfo { - v, sz := charInfoTrie.lookupString(s) - return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 8)} -} - -func lookupInfoNFKC(b []byte) runeInfo { - v, sz := charInfoTrie.lookup(b) - return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 12)} -} - -func lookupInfoStringNFKC(s string) runeInfo { - v, sz := charInfoTrie.lookupString(s) +func lookupInfoNFKC(b input, i int) runeInfo { + v, sz := b.charinfo(i) return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 12)} } diff --git a/libgo/go/exp/norm/input.go b/libgo/go/exp/norm/input.go new file mode 100644 index 0000000..12360a8 --- /dev/null +++ b/libgo/go/exp/norm/input.go @@ -0,0 +1,107 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "utf8" + +type input interface { + skipASCII(p int) int + skipNonStarter() int + appendSlice(buf []byte, s, e int) []byte + copySlice(buf []byte, s, e int) + charinfo(p int) (uint16, int) + decomposeNFC(p int) uint16 + decomposeNFKC(p int) uint16 + hangul(p int) uint32 +} + +type inputString string + +func (s inputString) skipASCII(p int) int { + for ; p < len(s) && s[p] < utf8.RuneSelf; p++ { + } + return p +} + +func (s inputString) skipNonStarter() int { + p := 0 + for ; p < len(s) && !utf8.RuneStart(s[p]); p++ { + } + return p +} + +func (s inputString) appendSlice(buf []byte, b, e int) []byte { + for i := b; i < e; i++ { + buf = append(buf, s[i]) + } + return buf +} + +func (s inputString) copySlice(buf []byte, b, e int) { + copy(buf, s[b:e]) +} + +func (s inputString) charinfo(p int) (uint16, int) { + return charInfoTrie.lookupString(string(s[p:])) +} + +func (s inputString) decomposeNFC(p int) uint16 { + return nfcDecompTrie.lookupStringUnsafe(string(s[p:])) +} + +func (s inputString) decomposeNFKC(p int) uint16 { + return nfkcDecompTrie.lookupStringUnsafe(string(s[p:])) +} + +func (s inputString) hangul(p int) uint32 { + if !isHangulString(string(s[p:])) { + return 0 + } + rune, _ := utf8.DecodeRuneInString(string(s[p:])) + return uint32(rune) +} + +type inputBytes []byte + +func (s inputBytes) skipASCII(p int) int { + for ; p < len(s) && s[p] < utf8.RuneSelf; p++ { + } + return p +} + +func (s inputBytes) skipNonStarter() int { + p := 0 + for ; p < len(s) && !utf8.RuneStart(s[p]); p++ { + } + return p +} + +func (s inputBytes) appendSlice(buf []byte, b, e int) []byte { + return append(buf, s[b:e]...) +} + +func (s inputBytes) copySlice(buf []byte, b, e int) { + copy(buf, s[b:e]) +} + +func (s inputBytes) charinfo(p int) (uint16, int) { + return charInfoTrie.lookup(s[p:]) +} + +func (s inputBytes) decomposeNFC(p int) uint16 { + return nfcDecompTrie.lookupUnsafe(s[p:]) +} + +func (s inputBytes) decomposeNFKC(p int) uint16 { + return nfkcDecompTrie.lookupUnsafe(s[p:]) +} + +func (s inputBytes) hangul(p int) uint32 { + if !isHangul(s[p:]) { + return 0 + } + rune, _ := utf8.DecodeRune(s[p:]) + return uint32(rune) +} diff --git a/libgo/go/exp/norm/maketables.go b/libgo/go/exp/norm/maketables.go index e3e5700a..14718c5 100644 --- a/libgo/go/exp/norm/maketables.go +++ b/libgo/go/exp/norm/maketables.go @@ -515,9 +515,13 @@ func completeCharFields(form int) { f.quickCheck[MComposed] = QCNo case (i & 0xffff00) == JamoLBase: f.quickCheck[MComposed] = QCYes + if JamoLBase <= i && i < JamoLEnd { + f.combinesForward = true + } if JamoVBase <= i && i < JamoVEnd { f.quickCheck[MComposed] = QCMaybe f.combinesBackward = true + f.combinesForward = true } if JamoTBase <= i && i < JamoTEnd { f.quickCheck[MComposed] = QCMaybe @@ -562,7 +566,7 @@ func makeEntry(f *FormInfo) uint16 { case QCMaybe: e |= 0x6 default: - log.Fatalf("Illegal quickcheck value %d.", f.quickCheck[MComposed]) + log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) } return e } diff --git a/libgo/go/exp/norm/maketesttables.go b/libgo/go/exp/norm/maketesttables.go index c5f6a64..fdcc114 100644 --- a/libgo/go/exp/norm/maketesttables.go +++ b/libgo/go/exp/norm/maketesttables.go @@ -21,6 +21,7 @@ var testRunes = []int{ 0x80, 0x100, 0x7FF, // 2-byte sequences 0x800, 0x999, 0xFFFF, // 3-byte sequences 0x10000, 0x10101, 0x10FFFF, // 4-byte sequences + 0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block } const fileHeader = `// Generated by running diff --git a/libgo/go/exp/norm/normalize.go b/libgo/go/exp/norm/normalize.go index e9d18dd..391bc41 100644 --- a/libgo/go/exp/norm/normalize.go +++ b/libgo/go/exp/norm/normalize.go @@ -5,6 +5,8 @@ // Package norm contains types and functions for normalizing Unicode strings. package norm +import "utf8" + // A Form denotes a canonical representation of Unicode code points. // The Unicode-defined normalization and equivalence forms are: // @@ -32,68 +34,431 @@ const ( // Bytes returns f(b). May return b if f(b) = b. func (f Form) Bytes(b []byte) []byte { - panic("not implemented") + n := f.QuickSpan(b) + if n == len(b) { + return b + } + out := make([]byte, n, len(b)) + copy(out, b[0:n]) + return f.Append(out, b[n:]...) } // String returns f(s). func (f Form) String(s string) string { - panic("not implemented") + n := f.QuickSpanString(s) + if n == len(s) { + return s + } + out := make([]byte, 0, len(s)) + copy(out, s[0:n]) + return string(f.AppendString(out, s[n:])) } // IsNormal returns true if b == f(b). func (f Form) IsNormal(b []byte) bool { - panic("not implemented") + rb := reorderBuffer{} + rb.init(f, b) + bp := quickSpan(&rb, 0) + if bp == len(b) { + return true + } + for bp < len(b) { + decomposeSegment(&rb, bp) + if rb.f.composing { + rb.compose() + } + for i := 0; i < rb.nrune; i++ { + info := rb.rune[i] + if bp+int(info.size) > len(b) { + return false + } + p := info.pos + pe := p + info.size + for ; p < pe; p++ { + if b[bp] != rb.byte[p] { + return false + } + bp++ + } + } + rb.reset() + bp = quickSpan(&rb, bp) + } + return true } // IsNormalString returns true if s == f(s). func (f Form) IsNormalString(s string) bool { - panic("not implemented") + rb := reorderBuffer{} + rb.initString(f, s) + bp := quickSpan(&rb, 0) + if bp == len(s) { + return true + } + for bp < len(s) { + decomposeSegment(&rb, bp) + if rb.f.composing { + rb.compose() + } + for i := 0; i < rb.nrune; i++ { + info := rb.rune[i] + if bp+int(info.size) > len(s) { + return false + } + p := info.pos + pe := p + info.size + for ; p < pe; p++ { + if s[bp] != rb.byte[p] { + return false + } + bp++ + } + } + rb.reset() + bp = quickSpan(&rb, bp) + } + return true +} + +// patchTail fixes a case where a rune may be incorrectly normalized +// if it is followed by illegal continuation bytes. It returns the +// patched buffer and the number of trailing continuation bytes that +// have been dropped. +func patchTail(rb *reorderBuffer, buf []byte) ([]byte, int) { + info, p := lastRuneStart(&rb.f, buf) + if p == -1 || info.size == 0 { + return buf, 0 + } + end := p + int(info.size) + extra := len(buf) - end + if extra > 0 { + buf = decomposeToLastBoundary(rb, buf[:end]) + if rb.f.composing { + rb.compose() + } + return rb.flush(buf), extra + } + return buf, 0 +} + +func appendQuick(rb *reorderBuffer, dst []byte, i int) ([]byte, int) { + if rb.nsrc == i { + return dst, i + } + end := quickSpan(rb, i) + return rb.src.appendSlice(dst, i, end), end } // Append returns f(append(out, b...)). -// The buffer out must be empty or equal to f(out). -func (f Form) Append(out, b []byte) []byte { - panic("not implemented") +// The buffer out must be nil, empty, or equal to f(out). +func (f Form) Append(out []byte, src ...byte) []byte { + if len(src) == 0 { + return out + } + rb := reorderBuffer{} + rb.init(f, src) + return doAppend(&rb, out) +} + +func doAppend(rb *reorderBuffer, out []byte) []byte { + src, n := rb.src, rb.nsrc + doMerge := len(out) > 0 + p := 0 + if p = src.skipNonStarter(); p > 0 { + // Move leading non-starters to destination. + out = src.appendSlice(out, 0, p) + buf, ndropped := patchTail(rb, out) + if ndropped > 0 { + out = src.appendSlice(buf, p-ndropped, p) + doMerge = false // no need to merge, ends with illegal UTF-8 + } else { + out = decomposeToLastBoundary(rb, buf) // force decomposition + } + } + fd := &rb.f + if doMerge { + var info runeInfo + if p < n { + info = fd.info(src, p) + if p == 0 && !fd.boundaryBefore(fd, info) { + out = decomposeToLastBoundary(rb, out) + } + } + if info.size == 0 || fd.boundaryBefore(fd, info) { + if fd.composing { + rb.compose() + } + out = rb.flush(out) + if info.size == 0 { + // Append incomplete UTF-8 encoding. + return src.appendSlice(out, p, n) + } + } + } + if rb.nrune == 0 { + out, p = appendQuick(rb, out, p) + } + for p < n { + p = decomposeSegment(rb, p) + if fd.composing { + rb.compose() + } + out = rb.flush(out) + out, p = appendQuick(rb, out, p) + } + return out } // AppendString returns f(append(out, []byte(s))). -// The buffer out must be empty or equal to f(out). -func (f Form) AppendString(out []byte, s string) []byte { - panic("not implemented") +// The buffer out must be nil, empty, or equal to f(out). +func (f Form) AppendString(out []byte, src string) []byte { + if len(src) == 0 { + return out + } + rb := reorderBuffer{} + rb.initString(f, src) + return doAppend(&rb, out) } // QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]). // It is not guaranteed to return the largest such n. func (f Form) QuickSpan(b []byte) int { - panic("not implemented") + rb := reorderBuffer{} + rb.init(f, b) + return quickSpan(&rb, 0) +} + +func quickSpan(rb *reorderBuffer, i int) int { + var lastCC uint8 + var nc int + lastSegStart := i + src, n := rb.src, rb.nsrc + for i < n { + if j := src.skipASCII(i); i != j { + i = j + lastSegStart = i - 1 + lastCC = 0 + nc = 0 + continue + } + info := rb.f.info(src, i) + if info.size == 0 { + // include incomplete runes + return n + } + cc := info.ccc + if rb.f.composing { + if !info.flags.isYesC() { + break + } + } else { + if !info.flags.isYesD() { + break + } + } + if cc == 0 { + lastSegStart = i + nc = 0 + } else { + if nc >= maxCombiningChars { + lastSegStart = i + lastCC = cc + nc = 1 + } else { + if lastCC > cc { + return lastSegStart + } + nc++ + } + } + lastCC = cc + i += int(info.size) + } + if i == n { + return n + } + if rb.f.composing { + return lastSegStart + } + return i } // QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]). // It is not guaranteed to return the largest such n. func (f Form) QuickSpanString(s string) int { - panic("not implemented") + rb := reorderBuffer{} + rb.initString(f, s) + return quickSpan(&rb, 0) +} + +// FirstBoundary returns the position i of the first boundary in b +// or -1 if b contains no boundary. +func (f Form) FirstBoundary(b []byte) int { + rb := reorderBuffer{} + rb.init(f, b) + return firstBoundary(&rb) +} + +func firstBoundary(rb *reorderBuffer) int { + src, nsrc := rb.src, rb.nsrc + i := src.skipNonStarter() + if i >= nsrc { + return -1 + } + fd := &rb.f + info := fd.info(src, i) + for n := 0; info.size != 0 && !fd.boundaryBefore(fd, info); { + i += int(info.size) + if n++; n >= maxCombiningChars { + return i + } + if i >= nsrc { + if !fd.boundaryAfter(fd, info) { + return -1 + } + return nsrc + } + info = fd.info(src, i) + } + if info.size == 0 { + return -1 + } + return i +} + +// FirstBoundaryInString returns the position i of the first boundary in s +// or -1 if s contains no boundary. +func (f Form) FirstBoundaryInString(s string) int { + rb := reorderBuffer{} + rb.initString(f, s) + return firstBoundary(&rb) +} + +// LastBoundary returns the position i of the last boundary in b +// or -1 if b contains no boundary. +func (f Form) LastBoundary(b []byte) int { + return lastBoundary(formTable[f], b) } -// FirstBoundary returns the position i of the first boundary in b. -// It returns len(b), false if b contains no boundaries. -func (f Form) FirstBoundary(b []byte) (i int, ok bool) { - panic("not implemented") +func lastBoundary(fd *formInfo, b []byte) int { + i := len(b) + info, p := lastRuneStart(fd, b) + if p == -1 { + return -1 + } + if info.size == 0 { // ends with incomplete rune + if p == 0 { // starts wtih incomplete rune + return -1 + } + i = p + info, p = lastRuneStart(fd, b[:i]) + if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter + return i + } + } + if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8 + return i + } + if fd.boundaryAfter(fd, info) { + return i + } + i = p + for n := 0; i >= 0 && !fd.boundaryBefore(fd, info); { + info, p = lastRuneStart(fd, b[:i]) + if n++; n >= maxCombiningChars { + return len(b) + } + if p+int(info.size) != i { + if p == -1 { // no boundary found + return -1 + } + return i // boundary after an illegal UTF-8 encoding + } + i = p + } + return i } -// FirstBoundaryInString return the position i of the first boundary in s. -// It returns len(s), false if s contains no boundaries. -func (f Form) FirstBoundaryInString(s string) (i int, ok bool) { - panic("not implemented") +// decomposeSegment scans the first segment in src into rb. +// It returns the number of bytes consumed from src. +// TODO(mpvl): consider inserting U+034f (Combining Grapheme Joiner) +// when we detect a sequence of 30+ non-starter chars. +func decomposeSegment(rb *reorderBuffer, sp int) int { + // Force one character to be consumed. + info := rb.f.info(rb.src, sp) + if info.size == 0 { + return 0 + } + for rb.insert(rb.src, sp, info) { + sp += int(info.size) + if sp >= rb.nsrc { + break + } + info = rb.f.info(rb.src, sp) + bound := rb.f.boundaryBefore(&rb.f, info) + if bound || info.size == 0 { + break + } + } + return sp } -// LastBoundaryIn returns the position i of the last boundary in b. -// It returns 0, false if b contains no boundary. -func (f Form) LastBoundary(b []byte) (i int, ok bool) { - panic("not implemented") +// lastRuneStart returns the runeInfo and position of the last +// rune in buf or the zero runeInfo and -1 if no rune was found. +func lastRuneStart(fd *formInfo, buf []byte) (runeInfo, int) { + p := len(buf) - 1 + for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- { + } + if p < 0 { + return runeInfo{0, 0, 0, 0}, -1 + } + return fd.info(inputBytes(buf), p), p } -// LastBoundaryInString returns the position i of the last boundary in s. -// It returns 0, false if s contains no boundary. -func (f Form) LastBoundaryInString(s string) (i int, ok bool) { - panic("not implemented") +// decomposeToLastBoundary finds an open segment at the end of the buffer +// and scans it into rb. Returns the buffer minus the last segment. +func decomposeToLastBoundary(rb *reorderBuffer, buf []byte) []byte { + fd := &rb.f + info, i := lastRuneStart(fd, buf) + if int(info.size) != len(buf)-i { + // illegal trailing continuation bytes + return buf + } + if rb.f.boundaryAfter(fd, info) { + return buf + } + var add [maxBackRunes]runeInfo // stores runeInfo in reverse order + add[0] = info + padd := 1 + n := 1 + p := len(buf) - int(info.size) + for ; p >= 0 && !rb.f.boundaryBefore(fd, info); p -= int(info.size) { + info, i = lastRuneStart(fd, buf[:p]) + if int(info.size) != p-i { + break + } + // Check that decomposition doesn't result in overflow. + if info.flags.hasDecomposition() { + dcomp := rb.f.decompose(inputBytes(buf), p-int(info.size)) + for i := 0; i < len(dcomp); { + inf := rb.f.info(inputBytes(dcomp), i) + i += int(inf.size) + n++ + } + } else { + n++ + } + if n > maxBackRunes { + break + } + add[padd] = info + padd++ + } + pp := p + for padd--; padd >= 0; padd-- { + info = add[padd] + rb.insert(inputBytes(buf), pp, info) + pp += int(info.size) + } + return buf[:p] } diff --git a/libgo/go/exp/norm/normalize_test.go b/libgo/go/exp/norm/normalize_test.go new file mode 100644 index 0000000..e374edf --- /dev/null +++ b/libgo/go/exp/norm/normalize_test.go @@ -0,0 +1,644 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "strings" + "testing" +) + +type PositionTest struct { + input string + pos int + buffer string // expected contents of reorderBuffer, if applicable +} + +type positionFunc func(rb *reorderBuffer, s string) int + +func runPosTests(t *testing.T, name string, f Form, fn positionFunc, tests []PositionTest) { + rb := reorderBuffer{} + rb.init(f, nil) + for i, test := range tests { + rb.reset() + rb.src = inputString(test.input) + rb.nsrc = len(test.input) + pos := fn(&rb, test.input) + if pos != test.pos { + t.Errorf("%s:%d: position is %d; want %d", name, i, pos, test.pos) + } + runes := []int(test.buffer) + if rb.nrune != len(runes) { + t.Errorf("%s:%d: reorder buffer lenght is %d; want %d", name, i, rb.nrune, len(runes)) + continue + } + for j, want := range runes { + found := int(rb.runeAt(j)) + if found != want { + t.Errorf("%s:%d: rune at %d is %U; want %U", name, i, j, found, want) + } + } + } +} + +var decomposeSegmentTests = []PositionTest{ + // illegal runes + {"\xC0", 0, ""}, + {"\u00E0\x80", 2, "\u0061\u0300"}, + // starter + {"a", 1, "a"}, + {"ab", 1, "a"}, + // starter + composing + {"a\u0300", 3, "a\u0300"}, + {"a\u0300b", 3, "a\u0300"}, + // with decomposition + {"\u00C0", 2, "A\u0300"}, + {"\u00C0b", 2, "A\u0300"}, + // long + {strings.Repeat("\u0300", 31), 62, strings.Repeat("\u0300", 31)}, + // ends with incomplete UTF-8 encoding + {"\xCC", 0, ""}, + {"\u0300\xCC", 2, "\u0300"}, +} + +func decomposeSegmentF(rb *reorderBuffer, s string) int { + rb.src = inputString(s) + rb.nsrc = len(s) + return decomposeSegment(rb, 0) +} + +func TestDecomposeSegment(t *testing.T) { + runPosTests(t, "TestDecomposeSegment", NFC, decomposeSegmentF, decomposeSegmentTests) +} + +var firstBoundaryTests = []PositionTest{ + // no boundary + {"", -1, ""}, + {"\u0300", -1, ""}, + {"\x80\x80", -1, ""}, + // illegal runes + {"\xff", 0, ""}, + {"\u0300\xff", 2, ""}, + {"\u0300\xc0\x80\x80", 2, ""}, + // boundaries + {"a", 0, ""}, + {"\u0300a", 2, ""}, + // Hangul + {"\u1103\u1161", 0, ""}, + {"\u110B\u1173\u11B7", 0, ""}, + {"\u1161\u110B\u1173\u11B7", 3, ""}, + {"\u1173\u11B7\u1103\u1161", 6, ""}, + // too many combining characters. + {strings.Repeat("\u0300", maxCombiningChars-1), -1, ""}, + {strings.Repeat("\u0300", maxCombiningChars), 60, ""}, + {strings.Repeat("\u0300", maxCombiningChars+1), 60, ""}, +} + +func firstBoundaryF(rb *reorderBuffer, s string) int { + return rb.f.form.FirstBoundary([]byte(s)) +} + +func firstBoundaryStringF(rb *reorderBuffer, s string) int { + return rb.f.form.FirstBoundaryInString(s) +} + +func TestFirstBoundary(t *testing.T) { + runPosTests(t, "TestFirstBoundary", NFC, firstBoundaryF, firstBoundaryTests) + runPosTests(t, "TestFirstBoundaryInString", NFC, firstBoundaryStringF, firstBoundaryTests) +} + +var decomposeToLastTests = []PositionTest{ + // ends with inert character + {"Hello!", 6, ""}, + {"\u0632", 2, ""}, + {"a\u0301\u0635", 5, ""}, + // ends with non-inert starter + {"a", 0, "a"}, + {"a\u0301a", 3, "a"}, + {"a\u0301\u03B9", 3, "\u03B9"}, + {"a\u0327", 0, "a\u0327"}, + // illegal runes + {"\xFF", 1, ""}, + {"aa\xFF", 3, ""}, + {"\xC0\x80\x80", 3, ""}, + {"\xCC\x80\x80", 3, ""}, + // ends with incomplete UTF-8 encoding + {"a\xCC", 2, ""}, + // ends with combining characters + {"\u0300\u0301", 0, "\u0300\u0301"}, + {"a\u0300\u0301", 0, "a\u0300\u0301"}, + {"a\u0301\u0308", 0, "a\u0301\u0308"}, + {"a\u0308\u0301", 0, "a\u0308\u0301"}, + {"aaaa\u0300\u0301", 3, "a\u0300\u0301"}, + {"\u0300a\u0300\u0301", 2, "a\u0300\u0301"}, + {"\u00C0", 0, "A\u0300"}, + {"a\u00C0", 1, "A\u0300"}, + // decomposing + {"a\u0300\uFDC0", 3, "\u0645\u062C\u064A"}, + {"\uFDC0" + strings.Repeat("\u0300", 26), 0, "\u0645\u062C\u064A" + strings.Repeat("\u0300", 26)}, + // Hangul + {"a\u1103", 1, "\u1103"}, + {"a\u110B", 1, "\u110B"}, + {"a\u110B\u1173", 1, "\u110B\u1173"}, + // See comment in composition.go:compBoundaryAfter. + {"a\u110B\u1173\u11B7", 1, "\u110B\u1173\u11B7"}, + {"a\uC73C", 1, "\u110B\u1173"}, + {"다음", 3, "\u110B\u1173\u11B7"}, + {"다", 0, "\u1103\u1161"}, + {"\u1103\u1161\u110B\u1173\u11B7", 6, "\u110B\u1173\u11B7"}, + {"\u110B\u1173\u11B7\u1103\u1161", 9, "\u1103\u1161"}, + {"다음음", 6, "\u110B\u1173\u11B7"}, + {"음다다", 6, "\u1103\u1161"}, + // buffer overflow + {"a" + strings.Repeat("\u0300", 30), 3, strings.Repeat("\u0300", 29)}, + {"\uFDFA" + strings.Repeat("\u0300", 14), 3, strings.Repeat("\u0300", 14)}, + // weird UTF-8 + {"a\u0300\u11B7", 0, "a\u0300\u11B7"}, +} + +func decomposeToLast(rb *reorderBuffer, s string) int { + buf := decomposeToLastBoundary(rb, []byte(s)) + return len(buf) +} + +func TestDecomposeToLastBoundary(t *testing.T) { + runPosTests(t, "TestDecomposeToLastBoundary", NFKC, decomposeToLast, decomposeToLastTests) +} + +var lastBoundaryTests = []PositionTest{ + // ends with inert character + {"Hello!", 6, ""}, + {"\u0632", 2, ""}, + // ends with non-inert starter + {"a", 0, ""}, + // illegal runes + {"\xff", 1, ""}, + {"aa\xff", 3, ""}, + {"a\xff\u0300", 1, ""}, + {"\xc0\x80\x80", 3, ""}, + {"\xc0\x80\x80\u0300", 3, ""}, + // ends with incomplete UTF-8 encoding + {"\xCC", -1, ""}, + {"\xE0\x80", -1, ""}, + {"\xF0\x80\x80", -1, ""}, + {"a\xCC", 0, ""}, + {"\x80\xCC", 1, ""}, + {"\xCC\xCC", 1, ""}, + // ends with combining characters + {"a\u0300\u0301", 0, ""}, + {"aaaa\u0300\u0301", 3, ""}, + {"\u0300a\u0300\u0301", 2, ""}, + {"\u00C0", 0, ""}, + {"a\u00C0", 1, ""}, + // decomposition may recombine + {"\u0226", 0, ""}, + // no boundary + {"", -1, ""}, + {"\u0300\u0301", -1, ""}, + {"\u0300", -1, ""}, + {"\x80\x80", -1, ""}, + {"\x80\x80\u0301", -1, ""}, + // Hangul + {"다음", 3, ""}, + {"다", 0, ""}, + {"\u1103\u1161\u110B\u1173\u11B7", 6, ""}, + {"\u110B\u1173\u11B7\u1103\u1161", 9, ""}, + // too many combining characters. + {strings.Repeat("\u0300", maxCombiningChars-1), -1, ""}, + {strings.Repeat("\u0300", maxCombiningChars), 60, ""}, + {strings.Repeat("\u0300", maxCombiningChars+1), 62, ""}, +} + +func lastBoundaryF(rb *reorderBuffer, s string) int { + return rb.f.form.LastBoundary([]byte(s)) +} + +func TestLastBoundary(t *testing.T) { + runPosTests(t, "TestLastBoundary", NFC, lastBoundaryF, lastBoundaryTests) +} + +var quickSpanTests = []PositionTest{ + {"", 0, ""}, + // starters + {"a", 1, ""}, + {"abc", 3, ""}, + {"\u043Eb", 3, ""}, + // incomplete last rune. + {"\xCC", 1, ""}, + {"a\xCC", 2, ""}, + // incorrectly ordered combining characters + {"\u0300\u0316", 0, ""}, + {"\u0300\u0316cd", 0, ""}, + // have a maximum number of combining characters. + {strings.Repeat("\u035D", 30) + "\u035B", 62, ""}, + {"a" + strings.Repeat("\u035D", 30) + "\u035B", 63, ""}, + {"Ɵ" + strings.Repeat("\u035D", 30) + "\u035B", 64, ""}, + {"aa" + strings.Repeat("\u035D", 30) + "\u035B", 64, ""}, +} + +var quickSpanNFDTests = []PositionTest{ + // needs decomposing + {"\u00C0", 0, ""}, + {"abc\u00C0", 3, ""}, + // correctly ordered combining characters + {"\u0300", 2, ""}, + {"ab\u0300", 4, ""}, + {"ab\u0300cd", 6, ""}, + {"\u0300cd", 4, ""}, + {"\u0316\u0300", 4, ""}, + {"ab\u0316\u0300", 6, ""}, + {"ab\u0316\u0300cd", 8, ""}, + {"ab\u0316\u0300\u00C0", 6, ""}, + {"\u0316\u0300cd", 6, ""}, + {"\u043E\u0308b", 5, ""}, + // incorrectly ordered combining characters + {"ab\u0300\u0316", 1, ""}, // TODO(mpvl): we could skip 'b' as well. + {"ab\u0300\u0316cd", 1, ""}, + // Hangul + {"같은", 0, ""}, +} + +var quickSpanNFCTests = []PositionTest{ + // okay composed + {"\u00C0", 2, ""}, + {"abc\u00C0", 5, ""}, + // correctly ordered combining characters + {"ab\u0300", 1, ""}, + {"ab\u0300cd", 1, ""}, + {"ab\u0316\u0300", 1, ""}, + {"ab\u0316\u0300cd", 1, ""}, + {"\u00C0\u035D", 4, ""}, + // we do not special case leading combining characters + {"\u0300cd", 0, ""}, + {"\u0300", 0, ""}, + {"\u0316\u0300", 0, ""}, + {"\u0316\u0300cd", 0, ""}, + // incorrectly ordered combining characters + {"ab\u0300\u0316", 1, ""}, + {"ab\u0300\u0316cd", 1, ""}, + // Hangul + {"같은", 6, ""}, +} + +func doQuickSpan(rb *reorderBuffer, s string) int { + return rb.f.form.QuickSpan([]byte(s)) +} + +func doQuickSpanString(rb *reorderBuffer, s string) int { + return rb.f.form.QuickSpanString(s) +} + +func TestQuickSpan(t *testing.T) { + runPosTests(t, "TestQuickSpanNFD1", NFD, doQuickSpan, quickSpanTests) + runPosTests(t, "TestQuickSpanNFD2", NFD, doQuickSpan, quickSpanNFDTests) + runPosTests(t, "TestQuickSpanNFC1", NFC, doQuickSpan, quickSpanTests) + runPosTests(t, "TestQuickSpanNFC2", NFC, doQuickSpan, quickSpanNFCTests) + + runPosTests(t, "TestQuickSpanStringNFD1", NFD, doQuickSpanString, quickSpanTests) + runPosTests(t, "TestQuickSpanStringNFD2", NFD, doQuickSpanString, quickSpanNFDTests) + runPosTests(t, "TestQuickSpanStringNFC1", NFC, doQuickSpanString, quickSpanTests) + runPosTests(t, "TestQuickSpanStringNFC2", NFC, doQuickSpanString, quickSpanNFCTests) +} + +var isNormalTests = []PositionTest{ + {"", 1, ""}, + // illegal runes + {"\xff", 1, ""}, + // starters + {"a", 1, ""}, + {"abc", 1, ""}, + {"\u043Eb", 1, ""}, + // incorrectly ordered combining characters + {"\u0300\u0316", 0, ""}, + {"ab\u0300\u0316", 0, ""}, + {"ab\u0300\u0316cd", 0, ""}, + {"\u0300\u0316cd", 0, ""}, +} +var isNormalNFDTests = []PositionTest{ + // needs decomposing + {"\u00C0", 0, ""}, + {"abc\u00C0", 0, ""}, + // correctly ordered combining characters + {"\u0300", 1, ""}, + {"ab\u0300", 1, ""}, + {"ab\u0300cd", 1, ""}, + {"\u0300cd", 1, ""}, + {"\u0316\u0300", 1, ""}, + {"ab\u0316\u0300", 1, ""}, + {"ab\u0316\u0300cd", 1, ""}, + {"\u0316\u0300cd", 1, ""}, + {"\u043E\u0308b", 1, ""}, + // Hangul + {"같은", 0, ""}, +} +var isNormalNFCTests = []PositionTest{ + // okay composed + {"\u00C0", 1, ""}, + {"abc\u00C0", 1, ""}, + // need reordering + {"a\u0300", 0, ""}, + {"a\u0300cd", 0, ""}, + {"a\u0316\u0300", 0, ""}, + {"a\u0316\u0300cd", 0, ""}, + // correctly ordered combining characters + {"ab\u0300", 1, ""}, + {"ab\u0300cd", 1, ""}, + {"ab\u0316\u0300", 1, ""}, + {"ab\u0316\u0300cd", 1, ""}, + {"\u00C0\u035D", 1, ""}, + {"\u0300", 1, ""}, + {"\u0316\u0300cd", 1, ""}, + // Hangul + {"같은", 1, ""}, +} + +func isNormalF(rb *reorderBuffer, s string) int { + if rb.f.form.IsNormal([]byte(s)) { + return 1 + } + return 0 +} + +func TestIsNormal(t *testing.T) { + runPosTests(t, "TestIsNormalNFD1", NFD, isNormalF, isNormalTests) + runPosTests(t, "TestIsNormalNFD2", NFD, isNormalF, isNormalNFDTests) + runPosTests(t, "TestIsNormalNFC1", NFC, isNormalF, isNormalTests) + runPosTests(t, "TestIsNormalNFC2", NFC, isNormalF, isNormalNFCTests) +} + +type AppendTest struct { + left string + right string + out string +} + +type appendFunc func(f Form, out []byte, s string) []byte + +func runAppendTests(t *testing.T, name string, f Form, fn appendFunc, tests []AppendTest) { + for i, test := range tests { + out := []byte(test.left) + out = fn(f, out, test.right) + outs := string(out) + if len(outs) != len(test.out) { + t.Errorf("%s:%d: length is %d; want %d", name, i, len(outs), len(test.out)) + } + if outs != test.out { + // Find first rune that differs and show context. + ir := []int(outs) + ig := []int(test.out) + for j := 0; j < len(ir) && j < len(ig); j++ { + if ir[j] == ig[j] { + continue + } + if j -= 3; j < 0 { + j = 0 + } + for e := j + 7; j < e && j < len(ir) && j < len(ig); j++ { + t.Errorf("%s:%d: runeAt(%d) = %U; want %U", name, i, j, ir[j], ig[j]) + } + break + } + } + } +} + +var appendTests = []AppendTest{ + // empty buffers + {"", "", ""}, + {"a", "", "a"}, + {"", "a", "a"}, + {"", "\u0041\u0307\u0304", "\u01E0"}, + // segment split across buffers + {"", "a\u0300b", "\u00E0b"}, + {"a", "\u0300b", "\u00E0b"}, + {"a", "\u0300\u0316", "\u00E0\u0316"}, + {"a", "\u0316\u0300", "\u00E0\u0316"}, + {"a", "\u0300a\u0300", "\u00E0\u00E0"}, + {"a", "\u0300a\u0300a\u0300", "\u00E0\u00E0\u00E0"}, + {"a", "\u0300aaa\u0300aaa\u0300", "\u00E0aa\u00E0aa\u00E0"}, + {"a\u0300", "\u0327", "\u00E0\u0327"}, + {"a\u0327", "\u0300", "\u00E0\u0327"}, + {"a\u0316", "\u0300", "\u00E0\u0316"}, + {"\u0041\u0307", "\u0304", "\u01E0"}, + // Hangul + {"", "\u110B\u1173", "\uC73C"}, + {"", "\u1103\u1161", "\uB2E4"}, + {"", "\u110B\u1173\u11B7", "\uC74C"}, + {"", "\u320E", "\x28\uAC00\x29"}, + {"", "\x28\u1100\u1161\x29", "\x28\uAC00\x29"}, + {"\u1103", "\u1161", "\uB2E4"}, + {"\u110B", "\u1173\u11B7", "\uC74C"}, + {"\u110B\u1173", "\u11B7", "\uC74C"}, + {"\uC73C", "\u11B7", "\uC74C"}, + // UTF-8 encoding split across buffers + {"a\xCC", "\x80", "\u00E0"}, + {"a\xCC", "\x80b", "\u00E0b"}, + {"a\xCC", "\x80a\u0300", "\u00E0\u00E0"}, + {"a\xCC", "\x80\x80", "\u00E0\x80"}, + {"a\xCC", "\x80\xCC", "\u00E0\xCC"}, + {"a\u0316\xCC", "\x80a\u0316\u0300", "\u00E0\u0316\u00E0\u0316"}, + // ending in incomplete UTF-8 encoding + {"", "\xCC", "\xCC"}, + {"a", "\xCC", "a\xCC"}, + {"a", "b\xCC", "ab\xCC"}, + {"\u0226", "\xCC", "\u0226\xCC"}, + // illegal runes + {"", "\x80", "\x80"}, + {"", "\x80\x80\x80", "\x80\x80\x80"}, + {"", "\xCC\x80\x80\x80", "\xCC\x80\x80\x80"}, + {"", "a\x80", "a\x80"}, + {"", "a\x80\x80\x80", "a\x80\x80\x80"}, + {"", "a\x80\x80\x80\x80\x80\x80", "a\x80\x80\x80\x80\x80\x80"}, + {"a", "\x80\x80\x80", "a\x80\x80\x80"}, + // overflow + {"", strings.Repeat("\x80", 33), strings.Repeat("\x80", 33)}, + {strings.Repeat("\x80", 33), "", strings.Repeat("\x80", 33)}, + {strings.Repeat("\x80", 33), strings.Repeat("\x80", 33), strings.Repeat("\x80", 66)}, + // overflow of combining characters + {strings.Repeat("\u0300", 33), "", strings.Repeat("\u0300", 33)}, + // weird UTF-8 + {"\u00E0\xE1", "\x86", "\u00E0\xE1\x86"}, + {"a\u0300\u11B7", "\u0300", "\u00E0\u11B7\u0300"}, + {"a\u0300\u11B7\u0300", "\u0300", "\u00E0\u11B7\u0300\u0300"}, + {"\u0300", "\xF8\x80\x80\x80\x80\u0300", "\u0300\xF8\x80\x80\x80\x80\u0300"}, + {"\u0300", "\xFC\x80\x80\x80\x80\x80\u0300", "\u0300\xFC\x80\x80\x80\x80\x80\u0300"}, + {"\xF8\x80\x80\x80\x80\u0300", "\u0300", "\xF8\x80\x80\x80\x80\u0300\u0300"}, + {"\xFC\x80\x80\x80\x80\x80\u0300", "\u0300", "\xFC\x80\x80\x80\x80\x80\u0300\u0300"}, +} + +func appendF(f Form, out []byte, s string) []byte { + return f.Append(out, []byte(s)...) +} + +func appendStringF(f Form, out []byte, s string) []byte { + return f.AppendString(out, s) +} + +func TestAppend(t *testing.T) { + runAppendTests(t, "TestAppend", NFKC, appendF, appendTests) + runAppendTests(t, "TestAppendString", NFKC, appendStringF, appendTests) +} + +func doFormBenchmark(b *testing.B, f Form, s string) { + b.StopTimer() + in := []byte(s) + buf := make([]byte, 2*len(in)) + b.SetBytes(int64(len(s))) + b.StartTimer() + for i := 0; i < b.N; i++ { + buf = f.Append(buf[0:0], in...) + buf = buf[0:0] + } +} + +var ascii = strings.Repeat("There is nothing to change here! ", 500) + +func BenchmarkNormalizeAsciiNFC(b *testing.B) { + doFormBenchmark(b, NFC, ascii) +} +func BenchmarkNormalizeAsciiNFD(b *testing.B) { + doFormBenchmark(b, NFD, ascii) +} +func BenchmarkNormalizeAsciiNFKC(b *testing.B) { + doFormBenchmark(b, NFKC, ascii) +} +func BenchmarkNormalizeAsciiNFKD(b *testing.B) { + doFormBenchmark(b, NFKD, ascii) +} + +func doTextBenchmark(b *testing.B, s string) { + b.StopTimer() + b.SetBytes(int64(len(s)) * 4) + in := []byte(s) + var buf = make([]byte, 0, 2*len(in)) + b.StartTimer() + for i := 0; i < b.N; i++ { + NFC.Append(buf, in...) + NFD.Append(buf, in...) + NFKC.Append(buf, in...) + NFKD.Append(buf, in...) + } +} + +func BenchmarkCanonicalOrdering(b *testing.B) { + doTextBenchmark(b, txt_canon) +} +func BenchmarkExtendedLatin(b *testing.B) { + doTextBenchmark(b, txt_vn) +} +func BenchmarkMiscTwoByteUtf8(b *testing.B) { + doTextBenchmark(b, twoByteUtf8) +} +func BenchmarkMiscThreeByteUtf8(b *testing.B) { + doTextBenchmark(b, threeByteUtf8) +} +func BenchmarkHangul(b *testing.B) { + doTextBenchmark(b, txt_kr) +} +func BenchmarkJapanese(b *testing.B) { + doTextBenchmark(b, txt_jp) +} +func BenchmarkChinese(b *testing.B) { + doTextBenchmark(b, txt_cn) +} + +// Tests sampled from the Canonical ordering tests (Part 2) of +// http://unicode.org/Public/UNIDATA/NormalizationTest.txt +const txt_canon = `\u0061\u0315\u0300\u05AE\u0300\u0062 \u0061\u0300\u0315\u0300\u05AE\u0062 +\u0061\u0302\u0315\u0300\u05AE\u0062 \u0061\u0307\u0315\u0300\u05AE\u0062 +\u0061\u0315\u0300\u05AE\u030A\u0062 \u0061\u059A\u0316\u302A\u031C\u0062 +\u0061\u032E\u059A\u0316\u302A\u0062 \u0061\u0338\u093C\u0334\u0062 +\u0061\u059A\u0316\u302A\u0339 \u0061\u0341\u0315\u0300\u05AE\u0062 +\u0061\u0348\u059A\u0316\u302A\u0062 \u0061\u0361\u0345\u035D\u035C\u0062 +\u0061\u0366\u0315\u0300\u05AE\u0062 \u0061\u0315\u0300\u05AE\u0486\u0062 +\u0061\u05A4\u059A\u0316\u302A\u0062 \u0061\u0315\u0300\u05AE\u0613\u0062 +\u0061\u0315\u0300\u05AE\u0615\u0062 \u0061\u0617\u0315\u0300\u05AE\u0062 +\u0061\u0619\u0618\u064D\u064E\u0062 \u0061\u0315\u0300\u05AE\u0654\u0062 +\u0061\u0315\u0300\u05AE\u06DC\u0062 \u0061\u0733\u0315\u0300\u05AE\u0062 +\u0061\u0744\u059A\u0316\u302A\u0062 \u0061\u0315\u0300\u05AE\u0745\u0062 +\u0061\u09CD\u05B0\u094D\u3099\u0062 \u0061\u0E38\u0E48\u0E38\u0C56\u0062 +\u0061\u0EB8\u0E48\u0E38\u0E49\u0062 \u0061\u0F72\u0F71\u0EC8\u0F71\u0062 +\u0061\u1039\u05B0\u094D\u3099\u0062 \u0061\u05B0\u094D\u3099\u1A60\u0062 +\u0061\u3099\u093C\u0334\u1BE6\u0062 \u0061\u3099\u093C\u0334\u1C37\u0062 +\u0061\u1CD9\u059A\u0316\u302A\u0062 \u0061\u2DED\u0315\u0300\u05AE\u0062 +\u0061\u2DEF\u0315\u0300\u05AE\u0062 \u0061\u302D\u302E\u059A\u0316\u0062` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/vn/ +const txt_vn = `Với các điều kiện sau: Ghi nhận công của tác giả. +Nếu bạn sử dụng, chuyển đổi, hoặc xây dựng dự án từ +nội dung được chia sẻ này, bạn phải áp dụng giấy phép này hoặc +một giấy phép khác có các điều khoản tương tự như giấy phép này +cho dự án của bạn. Hiểu rằng: Miễn — Bất kỳ các điều kiện nào +trên đây cũng có thể được miễn bỏ nếu bạn được sự cho phép của +người sở hữu bản quyền. Phạm vi công chúng — Khi tác phẩm hoặc +bất kỳ chương nào của tác phẩm đã trong vùng dành cho công +chúng theo quy định của pháp luật thì tình trạng của nó không +bị ảnh hưởng bởi giấy phép trong bất kỳ trường hợp nào.` + +// Taken from http://creativecommons.org/licenses/by-sa/1.0/deed.ru +const txt_ru = `При обязательном соблюдении следующих условий: +Attribution — Вы должны атрибутировать произведение (указывать +автора и источник) в порядке, предусмотренном автором или +лицензиаром (но только так, чтобы никоим образом не подразумевалось, +что они поддерживают вас или использование вами данного произведения). +Υπό τις ακόλουθες προϋποθέσεις:` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/gr/ +const txt_gr = `Αναφορά Δημιουργού — Θα πρέπει να κάνετε την αναφορά στο έργο με τον +τρόπο που έχει οριστεί από το δημιουργό ή το χορηγούντο την άδεια +(χωρίς όμως να εννοείται με οποιονδήποτε τρόπο ότι εγκρίνουν εσάς ή +τη χρήση του έργου από εσάς). Παρόμοια Διανομή — Εάν αλλοιώσετε, +τροποποιήσετε ή δημιουργήσετε περαιτέρω βασισμένοι στο έργο θα +μπορείτε να διανέμετε το έργο που θα προκύψει μόνο με την ίδια ή +παρόμοια άδεια.` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/deed.ar +const txt_ar = `بموجب الشروط التالية نسب المصنف — يجب عليك أن +تنسب العمل بالطريقة التي تحددها المؤلف أو المرخص (ولكن ليس بأي حال من +الأحوال أن توحي وتقترح بتحول أو استخدامك للعمل). +المشاركة على قدم المساواة — إذا كنت يعدل ، والتغيير ، أو الاستفادة +من هذا العمل ، قد ينتج عن توزيع العمل إلا في ظل تشابه او تطابق فى واحد +لهذا الترخيص.` + +// Taken from http://creativecommons.org/licenses/by-sa/1.0/il/ +const txt_il = `בכפוף לתנאים הבאים: ייחוס — עליך לייחס את היצירה (לתת קרדיט) באופן +המצויין על-ידי היוצר או מעניק הרישיון (אך לא בשום אופן המרמז על כך +שהם תומכים בך או בשימוש שלך ביצירה). שיתוף זהה — אם תחליט/י לשנות, +לעבד או ליצור יצירה נגזרת בהסתמך על יצירה זו, תוכל/י להפיץ את יצירתך +החדשה רק תחת אותו הרישיון או רישיון דומה לרישיון זה.` + +const twoByteUtf8 = txt_ru + txt_gr + txt_ar + txt_il + +// Taken from http://creativecommons.org/licenses/by-sa/2.0/kr/ +const txt_kr = `다음과 같은 조건을 따라야 합니다: 저작자표시 +(Attribution) — 저작자나 이용허락자가 정한 방법으로 저작물의 +원저작자를 표시하여야 합니다(그러나 원저작자가 이용자나 이용자의 +이용을 보증하거나 추천한다는 의미로 표시해서는 안됩니다). +동일조건변경허락 — 이 저작물을 이용하여 만든 이차적 저작물에는 본 +라이선스와 동일한 라이선스를 적용해야 합니다.` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/th/ +const txt_th = `ภายใต้เงื่อนไข ดังต่อไปนี้ : แสดงที่มา — คุณต้องแสดงที่ +มาของงานดังกล่าว ตามรูปแบบที่ผู้สร้างสรรค์หรือผู้อนุญาตกำหนด (แต่ +ไม่ใช่ในลักษณะที่ว่า พวกเขาสนับสนุนคุณหรือสนับสนุนการที่ +คุณนำงานไปใช้) อนุญาตแบบเดียวกัน — หากคุณดัดแปลง เปลี่ยนรูป หรื +อต่อเติมงานนี้ คุณต้องใช้สัญญาอนุญาตแบบเดียวกันหรือแบบที่เหมื +อนกับสัญญาอนุญาตที่ใช้กับงานนี้เท่านั้น` + +const threeByteUtf8 = txt_th + +// Taken from http://creativecommons.org/licenses/by-sa/2.0/jp/ +const txt_jp = `あなたの従うべき条件は以下の通りです。 +表示 — あなたは原著作者のクレジットを表示しなければなりません。 +継承 — もしあなたがこの作品を改変、変形または加工した場合、 +あなたはその結果生じた作品をこの作品と同一の許諾条件の下でのみ +頒布することができます。` + +// http://creativecommons.org/licenses/by-sa/2.5/cn/ +const txt_cn = `您可以自由： 复制、发行、展览、表演、放映、 +广播或通过信息网络传播本作品 创作演绎作品 +对本作品进行商业性使用 惟须遵守下列条件： +署名 — 您必须按照作者或者许可人指定的方式对作品进行署名。 +相同方式共享 — 如果您改变、转换本作品或者以本作品为基础进行创作， +您只能采用与本协议相同的许可协议发布基于本作品的演绎作品。` diff --git a/libgo/go/exp/norm/normregtest.go b/libgo/go/exp/norm/normregtest.go new file mode 100644 index 0000000..cbd73ff --- /dev/null +++ b/libgo/go/exp/norm/normregtest.go @@ -0,0 +1,295 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "exp/norm" + "flag" + "fmt" + "http" + "log" + "os" + "path" + "regexp" + "runtime" + "strings" + "strconv" + "time" + "utf8" +) + +func main() { + flag.Parse() + loadTestData() + CharacterByCharacterTests() + StandardTests() + PerformanceTest() + if errorCount == 0 { + fmt.Println("PASS") + } +} + +const file = "NormalizationTest.txt" + +var url = flag.String("url", + "http://www.unicode.org/Public/6.0.0/ucd/"+file, + "URL of Unicode database directory") +var localFiles = flag.Bool("local", + false, + "data files have been copied to the current directory; for debugging only") + +var logger = log.New(os.Stderr, "", log.Lshortfile) + +// This regression test runs the test set in NormalizationTest.txt +// (taken from http://www.unicode.org/Public/6.0.0/ucd/). +// +// NormalizationTest.txt has form: +// @Part0 # Specific cases +// # +// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE +// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW +// +// Each test has 5 columns (c1, c2, c3, c4, c5), where +// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1)) +// +// CONFORMANCE: +// 1. The following invariants must be true for all conformant implementations +// +// NFC +// c2 == NFC(c1) == NFC(c2) == NFC(c3) +// c4 == NFC(c4) == NFC(c5) +// +// NFD +// c3 == NFD(c1) == NFD(c2) == NFD(c3) +// c5 == NFD(c4) == NFD(c5) +// +// NFKC +// c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) +// +// NFKD +// c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) +// +// 2. For every code point X assigned in this version of Unicode that is not +// specifically listed in Part 1, the following invariants must be true +// for all conformant implementations: +// +// X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X) +// + +// Column types. +const ( + cRaw = iota + cNFC + cNFD + cNFKC + cNFKD + cMaxColumns +) + +// Holds data from NormalizationTest.txt +var part []Part + +type Part struct { + name string + number int + tests []Test +} + +type Test struct { + name string + partnr int + number int + rune int // used for character by character test + cols [cMaxColumns]string // Each has 5 entries, see below. +} + +func (t Test) Name() string { + if t.number < 0 { + return part[t.partnr].name + } + return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number) +} + +var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n$`) +var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n?$`) + +var counter int + +// Load the data form NormalizationTest.txt +func loadTestData() { + if *localFiles { + pwd, _ := os.Getwd() + *url = "file://" + path.Join(pwd, file) + } + t := &http.Transport{} + t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) + c := &http.Client{Transport: t} + resp, err := c.Get(*url) + if err != nil { + logger.Fatal(err) + } + if resp.StatusCode != 200 { + logger.Fatal("bad GET status for "+file, resp.Status) + } + f := resp.Body + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + if len(line) == 0 || line[0] == '#' { + continue + } + m := partRe.FindStringSubmatch(line) + if m != nil { + if len(m) < 3 { + logger.Fatal("Failed to parse Part: ", line) + } + i, err := strconv.Atoi(m[1]) + if err != nil { + logger.Fatal(err) + } + name := m[2] + part = append(part, Part{name: name[:len(name)-1], number: i}) + continue + } + m = testRe.FindStringSubmatch(line) + if m == nil || len(m) < 7 { + logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m) + } + test := Test{name: m[6], partnr: len(part) - 1, number: counter} + counter++ + for j := 1; j < len(m)-1; j++ { + for _, split := range strings.Split(m[j], " ") { + r, err := strconv.Btoui64(split, 16) + if err != nil { + logger.Fatal(err) + } + if test.rune == 0 { + // save for CharacterByCharacterTests + test.rune = int(r) + } + var buf [utf8.UTFMax]byte + sz := utf8.EncodeRune(buf[:], int(r)) + test.cols[j-1] += string(buf[:sz]) + } + } + part := &part[len(part)-1] + part.tests = append(part.tests, test) + } +} + +var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"} + +var errorCount int + +func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) { + if gold != result { + errorCount++ + if errorCount > 20 { + return + } + st, sr, sg := []int(test), []int(result), []int(gold) + logger.Printf("%s:%s: %s(%X)=%X; want:%X: %s", + t.Name(), name, fstr[f], st, sr, sg, t.name) + } +} + +func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) { + if result != want { + errorCount++ + if errorCount > 20 { + return + } + logger.Printf("%s:%s: %s(%X)=%v; want: %v", t.Name(), name, fstr[f], []int(test), result, want) + } +} + +func doTest(t *Test, f norm.Form, gold, test string) { + result := f.Bytes([]byte(test)) + cmpResult(t, "Bytes", f, gold, test, string(result)) + for i := range test { + out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...) + cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out)) + } + cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold) +} + +func doConformanceTests(t *Test, partn int) { + for i := 0; i <= 2; i++ { + doTest(t, norm.NFC, t.cols[1], t.cols[i]) + doTest(t, norm.NFD, t.cols[2], t.cols[i]) + doTest(t, norm.NFKC, t.cols[3], t.cols[i]) + doTest(t, norm.NFKD, t.cols[4], t.cols[i]) + } + for i := 3; i <= 4; i++ { + doTest(t, norm.NFC, t.cols[3], t.cols[i]) + doTest(t, norm.NFD, t.cols[4], t.cols[i]) + doTest(t, norm.NFKC, t.cols[3], t.cols[i]) + doTest(t, norm.NFKD, t.cols[4], t.cols[i]) + } +} + +func CharacterByCharacterTests() { + tests := part[1].tests + last := 0 + for i := 0; i <= len(tests); i++ { // last one is special case + var rune int + if i == len(tests) { + rune = 0x2FA1E // Don't have to go to 0x10FFFF + } else { + rune = tests[i].rune + } + for last++; last < rune; last++ { + // Check all characters that were not explicitly listed in the test. + t := &Test{partnr: 1, number: -1} + char := string(last) + doTest(t, norm.NFC, char, char) + doTest(t, norm.NFD, char, char) + doTest(t, norm.NFKC, char, char) + doTest(t, norm.NFKD, char, char) + } + if i < len(tests) { + doConformanceTests(&tests[i], 1) + } + } +} + +func StandardTests() { + for _, j := range []int{0, 2, 3} { + for _, test := range part[j].tests { + doConformanceTests(&test, j) + } + } +} + +// PerformanceTest verifies that normalization is O(n). If any of the +// code does not properly check for maxCombiningChars, normalization +// may exhibit O(n**2) behavior. +func PerformanceTest() { + runtime.GOMAXPROCS(2) + success := make(chan bool, 1) + go func() { + buf := bytes.Repeat([]byte("\u035D"), 1024*1024) + buf = append(buf, "\u035B"...) + norm.NFC.Append(nil, buf...) + success <- true + }() + timeout := time.After(1e9) + select { + case <-success: + // test completed before the timeout + case <-timeout: + errorCount++ + logger.Printf(`unexpectedly long time to complete PerformanceTest`) + } +} diff --git a/libgo/go/exp/norm/readwriter.go b/libgo/go/exp/norm/readwriter.go new file mode 100644 index 0000000..48ae135 --- /dev/null +++ b/libgo/go/exp/norm/readwriter.go @@ -0,0 +1,129 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "io" + "os" +) + +type normWriter struct { + rb reorderBuffer + w io.Writer + buf []byte +} + +// Write implements the standard write interface. If the last characters are +// not at a normalization boundary, the bytes will be buffered for the next +// write. The remaining bytes will be written on close. +func (w *normWriter) Write(data []byte) (n int, err os.Error) { + // Process data in pieces to keep w.buf size bounded. + const chunk = 4000 + + for len(data) > 0 { + // Normalize into w.buf. + m := len(data) + if m > chunk { + m = chunk + } + w.rb.src = inputBytes(data[:m]) + w.rb.nsrc = m + w.buf = doAppend(&w.rb, w.buf) + data = data[m:] + n += m + + // Write out complete prefix, save remainder. + // Note that lastBoundary looks back at most 30 runes. + i := lastBoundary(&w.rb.f, w.buf) + if i == -1 { + i = 0 + } + if i > 0 { + if _, err = w.w.Write(w.buf[:i]); err != nil { + break + } + bn := copy(w.buf, w.buf[i:]) + w.buf = w.buf[:bn] + } + } + return n, err +} + +// Close forces data that remains in the buffer to be written. +func (w *normWriter) Close() os.Error { + if len(w.buf) > 0 { + _, err := w.w.Write(w.buf) + if err != nil { + return err + } + } + return nil +} + +// Writer returns a new writer that implements Write(b) +// by writing f(b) to w. The returned writer may use an +// an internal buffer to maintain state across Write calls. +// Calling its Close method writes any buffered data to w. +func (f Form) Writer(w io.Writer) io.WriteCloser { + wr := &normWriter{rb: reorderBuffer{}, w: w} + wr.rb.init(f, nil) + return wr +} + +type normReader struct { + rb reorderBuffer + r io.Reader + inbuf []byte + outbuf []byte + bufStart int + lastBoundary int + err os.Error +} + +// Read implements the standard read interface. +func (r *normReader) Read(p []byte) (int, os.Error) { + for { + if r.lastBoundary-r.bufStart > 0 { + n := copy(p, r.outbuf[r.bufStart:r.lastBoundary]) + r.bufStart += n + if r.lastBoundary-r.bufStart > 0 { + return n, nil + } + return n, r.err + } + if r.err != nil { + return 0, r.err + } + outn := copy(r.outbuf, r.outbuf[r.lastBoundary:]) + r.outbuf = r.outbuf[0:outn] + r.bufStart = 0 + + n, err := r.r.Read(r.inbuf) + r.rb.src = inputBytes(r.inbuf[0:n]) + r.rb.nsrc, r.err = n, err + if n > 0 { + r.outbuf = doAppend(&r.rb, r.outbuf) + } + if err == os.EOF { + r.lastBoundary = len(r.outbuf) + } else { + r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf) + if r.lastBoundary == -1 { + r.lastBoundary = 0 + } + } + } + panic("should not reach here") +} + +// Reader returns a new reader that implements Read +// by reading data from r and returning f(data). +func (f Form) Reader(r io.Reader) io.Reader { + const chunk = 4000 + buf := make([]byte, chunk) + rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf} + rr.rb.init(f, buf) + return rr +} diff --git a/libgo/go/exp/norm/readwriter_test.go b/libgo/go/exp/norm/readwriter_test.go new file mode 100644 index 0000000..68652ef --- /dev/null +++ b/libgo/go/exp/norm/readwriter_test.go @@ -0,0 +1,69 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" +) + +var ioTests = []AppendTest{ + {"", strings.Repeat("a\u0316\u0300", 6), strings.Repeat("\u00E0\u0316", 6)}, + {"", strings.Repeat("a\u0300\u0316", 4000), strings.Repeat("\u00E0\u0316", 4000)}, + {"", strings.Repeat("\x80\x80", 4000), strings.Repeat("\x80\x80", 4000)}, + {"", "\u0041\u0307\u0304", "\u01E0"}, +} + +var bufSizes = []int{1, 2, 3, 4, 5, 6, 7, 8, 100, 101, 102, 103, 4000, 4001, 4002, 4003} + +func readFunc(size int) appendFunc { + return func(f Form, out []byte, s string) []byte { + out = append(out, s...) + r := f.Reader(bytes.NewBuffer(out)) + buf := make([]byte, size) + result := []byte{} + for n, err := 0, os.Error(nil); err == nil; { + n, err = r.Read(buf) + result = append(result, buf[:n]...) + } + return result + } +} + +func TestReader(t *testing.T) { + for _, s := range bufSizes { + name := fmt.Sprintf("TestReader%da", s) + runAppendTests(t, name, NFKC, readFunc(s), appendTests) + name = fmt.Sprintf("TestReader%db", s) + runAppendTests(t, name, NFKC, readFunc(s), ioTests) + } +} + +func writeFunc(size int) appendFunc { + return func(f Form, out []byte, s string) []byte { + in := append(out, s...) + result := new(bytes.Buffer) + w := f.Writer(result) + buf := make([]byte, size) + for n := 0; len(in) > 0; in = in[n:] { + n = copy(buf, in) + _, _ = w.Write(buf[:n]) + } + w.Close() + return result.Bytes() + } +} + +func TestWriter(t *testing.T) { + for _, s := range bufSizes { + name := fmt.Sprintf("TestWriter%da", s) + runAppendTests(t, name, NFKC, writeFunc(s), appendTests) + name = fmt.Sprintf("TestWriter%db", s) + runAppendTests(t, name, NFKC, writeFunc(s), ioTests) + } +} diff --git a/libgo/go/exp/norm/tables.go b/libgo/go/exp/norm/tables.go index 76995c2..55ff052 100644 --- a/libgo/go/exp/norm/tables.go +++ b/libgo/go/exp/norm/tables.go @@ -2490,545 +2490,632 @@ var decomps = [...]byte{ 0x98, 0x80, } -// nfcDecompValues: 4992 entries, 9984 bytes +// nfcDecompValues: 1408 entries, 2816 bytes // Block 2 is the null block. -var nfcDecompValues = [4992]uint16{ +var nfcDecompValues = [1408]uint16{ // Block 0x0, offset 0x0 // Block 0x1, offset 0x40 // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0x00c0: 0x0032, 0x00c1: 0x0036, 0x00c2: 0x003a, 0x00c3: 0x003e, 0x00c4: 0x0042, 0x00c5: 0x0046, - 0x00c7: 0x004a, 0x00c8: 0x004e, 0x00c9: 0x0052, 0x00ca: 0x0056, 0x00cb: 0x005a, - 0x00cc: 0x005e, 0x00cd: 0x0062, 0x00ce: 0x0066, 0x00cf: 0x006a, 0x00d1: 0x006e, - 0x00d2: 0x0072, 0x00d3: 0x0076, 0x00d4: 0x007a, 0x00d5: 0x007e, 0x00d6: 0x0082, - 0x00d9: 0x0086, 0x00da: 0x008a, 0x00db: 0x008e, 0x00dc: 0x0092, 0x00dd: 0x0096, - 0x00e0: 0x009a, 0x00e1: 0x009e, 0x00e2: 0x00a2, 0x00e3: 0x00a6, - 0x00e4: 0x00aa, 0x00e5: 0x00ae, 0x00e7: 0x00b2, 0x00e8: 0x00b6, 0x00e9: 0x00ba, - 0x00ea: 0x00be, 0x00eb: 0x00c2, 0x00ec: 0x00c6, 0x00ed: 0x00ca, 0x00ee: 0x00ce, 0x00ef: 0x00d2, - 0x00f1: 0x00d6, 0x00f2: 0x00da, 0x00f3: 0x00de, 0x00f4: 0x00e2, 0x00f5: 0x00e6, - 0x00f6: 0x00ea, 0x00f9: 0x00ee, 0x00fa: 0x00f2, 0x00fb: 0x00f6, - 0x00fc: 0x00fa, 0x00fd: 0x00fe, 0x00ff: 0x0102, + 0x00cd: 0x02fb, 0x00ce: 0x02ff, 0x00cf: 0x0303, 0x00d0: 0x0307, 0x00d1: 0x030b, + 0x00d2: 0x030f, 0x00d3: 0x0313, 0x00d4: 0x0317, 0x00d5: 0x031b, 0x00d6: 0x0321, 0x00d7: 0x0327, + 0x00d8: 0x032d, 0x00d9: 0x0333, 0x00da: 0x0339, 0x00db: 0x033f, 0x00dc: 0x0345, + 0x00de: 0x034b, 0x00df: 0x0351, 0x00e0: 0x0357, 0x00e1: 0x035d, 0x00e2: 0x0363, 0x00e3: 0x0368, + 0x00e6: 0x036d, 0x00e7: 0x0371, 0x00e8: 0x0375, 0x00e9: 0x0379, + 0x00ea: 0x037d, 0x00eb: 0x0381, 0x00ec: 0x0385, 0x00ed: 0x038b, 0x00ee: 0x0391, 0x00ef: 0x0396, + 0x00f0: 0x039b, 0x00f4: 0x03a8, 0x00f5: 0x03ac, + 0x00f8: 0x03b0, 0x00f9: 0x03b4, 0x00fa: 0x03b8, 0x00fb: 0x03be, + 0x00fc: 0x03c4, 0x00fd: 0x03c9, 0x00fe: 0x03ce, 0x00ff: 0x03d3, // Block 0x4, offset 0x100 - 0x0100: 0x0106, 0x0101: 0x010a, 0x0102: 0x010e, 0x0103: 0x0112, 0x0104: 0x0116, 0x0105: 0x011a, - 0x0106: 0x011e, 0x0107: 0x0122, 0x0108: 0x0126, 0x0109: 0x012a, 0x010a: 0x012e, 0x010b: 0x0132, - 0x010c: 0x0136, 0x010d: 0x013a, 0x010e: 0x013e, 0x010f: 0x0142, - 0x0112: 0x0146, 0x0113: 0x014a, 0x0114: 0x014e, 0x0115: 0x0152, 0x0116: 0x0156, 0x0117: 0x015a, - 0x0118: 0x015e, 0x0119: 0x0162, 0x011a: 0x0166, 0x011b: 0x016a, 0x011c: 0x016e, 0x011d: 0x0172, - 0x011e: 0x0176, 0x011f: 0x017a, 0x0120: 0x017e, 0x0121: 0x0182, 0x0122: 0x0186, 0x0123: 0x018a, - 0x0124: 0x018e, 0x0125: 0x0192, 0x0128: 0x0196, 0x0129: 0x019a, - 0x012a: 0x019e, 0x012b: 0x01a2, 0x012c: 0x01a6, 0x012d: 0x01aa, 0x012e: 0x01ae, 0x012f: 0x01b2, - 0x0130: 0x01b6, 0x0134: 0x01c0, 0x0135: 0x01c4, - 0x0136: 0x01c8, 0x0137: 0x01cc, 0x0139: 0x01d0, 0x013a: 0x01d4, 0x013b: 0x01d8, - 0x013c: 0x01dc, 0x013d: 0x01e0, 0x013e: 0x01e4, + 0x0100: 0x0b02, 0x0101: 0x0b06, 0x0102: 0x0b0a, 0x0103: 0x0b0e, 0x0104: 0x0b12, 0x0105: 0x0b16, + 0x0106: 0x0b1a, 0x0107: 0x0b1e, 0x0108: 0x0b22, 0x0109: 0x0b26, 0x010a: 0x0b2a, 0x010b: 0x0b2e, + 0x010c: 0x0b32, 0x010d: 0x0b38, 0x010e: 0x0b3e, 0x010f: 0x0b44, 0x0110: 0x0b4a, 0x0111: 0x0b50, + 0x0112: 0x0b56, 0x0113: 0x0b5c, 0x0114: 0x0b62, 0x0115: 0x0b66, 0x0116: 0x0b6a, 0x0117: 0x0b6e, + 0x0118: 0x0b72, 0x0119: 0x0b76, 0x011a: 0x0b7a, 0x011b: 0x0b7e, 0x011c: 0x0b82, 0x011d: 0x0b88, + 0x011e: 0x0b8e, 0x011f: 0x0b92, 0x0120: 0x0b96, 0x0121: 0x0b9a, 0x0122: 0x0b9e, 0x0123: 0x0ba2, + 0x0124: 0x0ba6, 0x0125: 0x0bac, 0x0126: 0x0bb2, 0x0127: 0x0bb8, 0x0128: 0x0bbe, 0x0129: 0x0bc4, + 0x012a: 0x0bca, 0x012b: 0x0bce, 0x012c: 0x0bd2, 0x012d: 0x0bd6, 0x012e: 0x0bda, 0x012f: 0x0bde, + 0x0130: 0x0be2, 0x0131: 0x0be6, 0x0132: 0x0bea, 0x0133: 0x0bee, 0x0134: 0x0bf2, 0x0135: 0x0bf6, + 0x0136: 0x0bfa, 0x0137: 0x0bfe, 0x0138: 0x0c02, 0x0139: 0x0c08, 0x013a: 0x0c0e, 0x013b: 0x0c14, + 0x013c: 0x0c1a, 0x013d: 0x0c1e, 0x013e: 0x0c22, 0x013f: 0x0c26, // Block 0x5, offset 0x140 - 0x0143: 0x01f0, 0x0144: 0x01f4, 0x0145: 0x01f8, - 0x0146: 0x01fc, 0x0147: 0x0200, 0x0148: 0x0204, - 0x014c: 0x020c, 0x014d: 0x0210, 0x014e: 0x0214, 0x014f: 0x0218, 0x0150: 0x021c, 0x0151: 0x0220, - 0x0154: 0x0224, 0x0155: 0x0228, 0x0156: 0x022c, 0x0157: 0x0230, - 0x0158: 0x0234, 0x0159: 0x0238, 0x015a: 0x023c, 0x015b: 0x0240, 0x015c: 0x0244, 0x015d: 0x0248, - 0x015e: 0x024c, 0x015f: 0x0250, 0x0160: 0x0254, 0x0161: 0x0258, 0x0162: 0x025c, 0x0163: 0x0260, - 0x0164: 0x0264, 0x0165: 0x0268, 0x0168: 0x026c, 0x0169: 0x0270, - 0x016a: 0x0274, 0x016b: 0x0278, 0x016c: 0x027c, 0x016d: 0x0280, 0x016e: 0x0284, 0x016f: 0x0288, - 0x0170: 0x028c, 0x0171: 0x0290, 0x0172: 0x0294, 0x0173: 0x0298, 0x0174: 0x029c, 0x0175: 0x02a0, - 0x0176: 0x02a4, 0x0177: 0x02a8, 0x0178: 0x02ac, 0x0179: 0x02b0, 0x017a: 0x02b4, 0x017b: 0x02b8, - 0x017c: 0x02bc, 0x017d: 0x02c0, 0x017e: 0x02c4, + 0x0140: 0x0c2a, 0x0141: 0x0c2e, 0x0142: 0x0c32, 0x0143: 0x0c36, 0x0144: 0x0c3a, 0x0145: 0x0c3e, + 0x0146: 0x0c42, 0x0147: 0x0c46, 0x0148: 0x0c4a, 0x0149: 0x0c4e, 0x014a: 0x0c52, 0x014b: 0x0c56, + 0x014c: 0x0c5a, 0x014d: 0x0c5e, 0x014e: 0x0c62, 0x014f: 0x0c66, 0x0150: 0x0c6a, 0x0151: 0x0c6e, + 0x0152: 0x0c72, 0x0153: 0x0c76, 0x0154: 0x0c7a, 0x0155: 0x0c7e, 0x0156: 0x0c82, 0x0157: 0x0c86, + 0x0158: 0x0c8a, 0x0159: 0x0c8e, 0x015b: 0x0c96, + 0x0160: 0x0c9b, 0x0161: 0x0c9f, 0x0162: 0x0ca3, 0x0163: 0x0ca7, + 0x0164: 0x0cab, 0x0165: 0x0cb1, 0x0166: 0x0cb7, 0x0167: 0x0cbd, 0x0168: 0x0cc3, 0x0169: 0x0cc9, + 0x016a: 0x0ccf, 0x016b: 0x0cd5, 0x016c: 0x0cdb, 0x016d: 0x0ce1, 0x016e: 0x0ce7, 0x016f: 0x0ced, + 0x0170: 0x0cf3, 0x0171: 0x0cf9, 0x0172: 0x0cff, 0x0173: 0x0d05, 0x0174: 0x0d0b, 0x0175: 0x0d11, + 0x0176: 0x0d17, 0x0177: 0x0d1d, 0x0178: 0x0d23, 0x0179: 0x0d27, 0x017a: 0x0d2b, 0x017b: 0x0d2f, + 0x017c: 0x0d33, 0x017d: 0x0d37, 0x017e: 0x0d3b, 0x017f: 0x0d41, // Block 0x6, offset 0x180 - 0x01a0: 0x02ca, 0x01a1: 0x02ce, - 0x01af: 0x02d2, - 0x01b0: 0x02d6, + 0x0180: 0x0d47, 0x0181: 0x0d4d, 0x0182: 0x0d53, 0x0183: 0x0d59, 0x0184: 0x0d5f, 0x0185: 0x0d65, + 0x0186: 0x0d6b, 0x0187: 0x0d71, 0x0188: 0x0d77, 0x0189: 0x0d7b, 0x018a: 0x0d7f, 0x018b: 0x0d83, + 0x018c: 0x0d87, 0x018d: 0x0d8b, 0x018e: 0x0d8f, 0x018f: 0x0d93, 0x0190: 0x0d97, 0x0191: 0x0d9d, + 0x0192: 0x0da3, 0x0193: 0x0da9, 0x0194: 0x0daf, 0x0195: 0x0db5, 0x0196: 0x0dbb, 0x0197: 0x0dc1, + 0x0198: 0x0dc7, 0x0199: 0x0dcd, 0x019a: 0x0dd3, 0x019b: 0x0dd9, 0x019c: 0x0ddf, 0x019d: 0x0de5, + 0x019e: 0x0deb, 0x019f: 0x0df1, 0x01a0: 0x0df7, 0x01a1: 0x0dfd, 0x01a2: 0x0e03, 0x01a3: 0x0e09, + 0x01a4: 0x0e0f, 0x01a5: 0x0e13, 0x01a6: 0x0e17, 0x01a7: 0x0e1b, 0x01a8: 0x0e1f, 0x01a9: 0x0e25, + 0x01aa: 0x0e2b, 0x01ab: 0x0e31, 0x01ac: 0x0e37, 0x01ad: 0x0e3d, 0x01ae: 0x0e43, 0x01af: 0x0e49, + 0x01b0: 0x0e4f, 0x01b1: 0x0e55, 0x01b2: 0x0e5b, 0x01b3: 0x0e5f, 0x01b4: 0x0e63, 0x01b5: 0x0e67, + 0x01b6: 0x0e6b, 0x01b7: 0x0e6f, 0x01b8: 0x0e73, 0x01b9: 0x0e77, // Block 0x7, offset 0x1c0 - 0x01cd: 0x02fb, 0x01ce: 0x02ff, 0x01cf: 0x0303, 0x01d0: 0x0307, 0x01d1: 0x030b, - 0x01d2: 0x030f, 0x01d3: 0x0313, 0x01d4: 0x0317, 0x01d5: 0x031b, 0x01d6: 0x0321, 0x01d7: 0x0327, - 0x01d8: 0x032d, 0x01d9: 0x0333, 0x01da: 0x0339, 0x01db: 0x033f, 0x01dc: 0x0345, - 0x01de: 0x034b, 0x01df: 0x0351, 0x01e0: 0x0357, 0x01e1: 0x035d, 0x01e2: 0x0363, 0x01e3: 0x0368, - 0x01e6: 0x036d, 0x01e7: 0x0371, 0x01e8: 0x0375, 0x01e9: 0x0379, - 0x01ea: 0x037d, 0x01eb: 0x0381, 0x01ec: 0x0385, 0x01ed: 0x038b, 0x01ee: 0x0391, 0x01ef: 0x0396, - 0x01f0: 0x039b, 0x01f4: 0x03a8, 0x01f5: 0x03ac, - 0x01f8: 0x03b0, 0x01f9: 0x03b4, 0x01fa: 0x03b8, 0x01fb: 0x03be, - 0x01fc: 0x03c4, 0x01fd: 0x03c9, 0x01fe: 0x03ce, 0x01ff: 0x03d3, + 0x01c0: 0x0e7b, 0x01c1: 0x0e80, 0x01c2: 0x0e85, 0x01c3: 0x0e8c, 0x01c4: 0x0e93, 0x01c5: 0x0e9a, + 0x01c6: 0x0ea1, 0x01c7: 0x0ea8, 0x01c8: 0x0eaf, 0x01c9: 0x0eb4, 0x01ca: 0x0eb9, 0x01cb: 0x0ec0, + 0x01cc: 0x0ec7, 0x01cd: 0x0ece, 0x01ce: 0x0ed5, 0x01cf: 0x0edc, 0x01d0: 0x0ee3, 0x01d1: 0x0ee8, + 0x01d2: 0x0eed, 0x01d3: 0x0ef4, 0x01d4: 0x0efb, 0x01d5: 0x0f02, + 0x01d8: 0x0f09, 0x01d9: 0x0f0e, 0x01da: 0x0f13, 0x01db: 0x0f1a, 0x01dc: 0x0f21, 0x01dd: 0x0f28, + 0x01e0: 0x0f2f, 0x01e1: 0x0f34, 0x01e2: 0x0f39, 0x01e3: 0x0f40, + 0x01e4: 0x0f47, 0x01e5: 0x0f4e, 0x01e6: 0x0f55, 0x01e7: 0x0f5c, 0x01e8: 0x0f63, 0x01e9: 0x0f68, + 0x01ea: 0x0f6d, 0x01eb: 0x0f74, 0x01ec: 0x0f7b, 0x01ed: 0x0f82, 0x01ee: 0x0f89, 0x01ef: 0x0f90, + 0x01f0: 0x0f97, 0x01f1: 0x0f9c, 0x01f2: 0x0fa1, 0x01f3: 0x0fa8, 0x01f4: 0x0faf, 0x01f5: 0x0fb6, + 0x01f6: 0x0fbd, 0x01f7: 0x0fc4, 0x01f8: 0x0fcb, 0x01f9: 0x0fd0, 0x01fa: 0x0fd5, 0x01fb: 0x0fdc, + 0x01fc: 0x0fe3, 0x01fd: 0x0fea, 0x01fe: 0x0ff1, 0x01ff: 0x0ff8, // Block 0x8, offset 0x200 - 0x0200: 0x03d8, 0x0201: 0x03dc, 0x0202: 0x03e0, 0x0203: 0x03e4, 0x0204: 0x03e8, 0x0205: 0x03ec, - 0x0206: 0x03f0, 0x0207: 0x03f4, 0x0208: 0x03f8, 0x0209: 0x03fc, 0x020a: 0x0400, 0x020b: 0x0404, - 0x020c: 0x0408, 0x020d: 0x040c, 0x020e: 0x0410, 0x020f: 0x0414, 0x0210: 0x0418, 0x0211: 0x041c, - 0x0212: 0x0420, 0x0213: 0x0424, 0x0214: 0x0428, 0x0215: 0x042c, 0x0216: 0x0430, 0x0217: 0x0434, - 0x0218: 0x0438, 0x0219: 0x043c, 0x021a: 0x0440, 0x021b: 0x0444, - 0x021e: 0x0448, 0x021f: 0x044c, - 0x0226: 0x0450, 0x0227: 0x0454, 0x0228: 0x0458, 0x0229: 0x045c, - 0x022a: 0x0460, 0x022b: 0x0466, 0x022c: 0x046c, 0x022d: 0x0472, 0x022e: 0x0478, 0x022f: 0x047c, - 0x0230: 0x0480, 0x0231: 0x0486, 0x0232: 0x048c, 0x0233: 0x0490, + 0x0200: 0x0fff, 0x0201: 0x1004, 0x0202: 0x1009, 0x0203: 0x1010, 0x0204: 0x1017, 0x0205: 0x101e, + 0x0208: 0x1025, 0x0209: 0x102a, 0x020a: 0x102f, 0x020b: 0x1036, + 0x020c: 0x103d, 0x020d: 0x1044, 0x0210: 0x104b, 0x0211: 0x1050, + 0x0212: 0x1055, 0x0213: 0x105c, 0x0214: 0x1063, 0x0215: 0x106a, 0x0216: 0x1071, 0x0217: 0x1078, + 0x0219: 0x107f, 0x021b: 0x1084, 0x021d: 0x108b, + 0x021f: 0x1092, 0x0220: 0x1099, 0x0221: 0x109e, 0x0222: 0x10a3, 0x0223: 0x10aa, + 0x0224: 0x10b1, 0x0225: 0x10b8, 0x0226: 0x10bf, 0x0227: 0x10c6, 0x0228: 0x10cd, 0x0229: 0x10d2, + 0x022a: 0x10d7, 0x022b: 0x10de, 0x022c: 0x10e5, 0x022d: 0x10ec, 0x022e: 0x10f3, 0x022f: 0x10fa, + 0x0230: 0x1101, 0x0231: 0x0525, 0x0232: 0x1106, 0x0233: 0x052a, 0x0234: 0x110b, 0x0235: 0x052f, + 0x0236: 0x1110, 0x0237: 0x0534, 0x0238: 0x1115, 0x0239: 0x054a, 0x023a: 0x111a, 0x023b: 0x054f, + 0x023c: 0x111f, 0x023d: 0x0554, // Block 0x9, offset 0x240 - 0x0240: 0x04cc, 0x0241: 0x04cf, 0x0243: 0x04d2, 0x0244: 0x04d5, - 0x0274: 0x04da, - 0x027e: 0x04e1, + 0x0240: 0x1124, 0x0241: 0x112b, 0x0242: 0x1132, 0x0243: 0x113b, 0x0244: 0x1144, 0x0245: 0x114d, + 0x0246: 0x1156, 0x0247: 0x115f, 0x0248: 0x1168, 0x0249: 0x116f, 0x024a: 0x1176, 0x024b: 0x117f, + 0x024c: 0x1188, 0x024d: 0x1191, 0x024e: 0x119a, 0x024f: 0x11a3, 0x0250: 0x11ac, 0x0251: 0x11b3, + 0x0252: 0x11ba, 0x0253: 0x11c3, 0x0254: 0x11cc, 0x0255: 0x11d5, 0x0256: 0x11de, 0x0257: 0x11e7, + 0x0258: 0x11f0, 0x0259: 0x11f7, 0x025a: 0x11fe, 0x025b: 0x1207, 0x025c: 0x1210, 0x025d: 0x1219, + 0x025e: 0x1222, 0x025f: 0x122b, 0x0260: 0x1234, 0x0261: 0x123b, 0x0262: 0x1242, 0x0263: 0x124b, + 0x0264: 0x1254, 0x0265: 0x125d, 0x0266: 0x1266, 0x0267: 0x126f, 0x0268: 0x1278, 0x0269: 0x127f, + 0x026a: 0x1286, 0x026b: 0x128f, 0x026c: 0x1298, 0x026d: 0x12a1, 0x026e: 0x12aa, 0x026f: 0x12b3, + 0x0270: 0x12bc, 0x0271: 0x12c1, 0x0272: 0x12c6, 0x0273: 0x12cd, 0x0274: 0x12d2, + 0x0276: 0x12d9, 0x0277: 0x12de, 0x0278: 0x12e5, 0x0279: 0x12ea, 0x027a: 0x12ef, 0x027b: 0x04ee, + 0x027c: 0x12f4, 0x027e: 0x12fd, // Block 0xa, offset 0x280 - 0x0285: 0x04e3, - 0x0286: 0x04ee, 0x0287: 0x04f3, 0x0288: 0x04f6, 0x0289: 0x04fb, 0x028a: 0x0500, - 0x028c: 0x0505, 0x028e: 0x050a, 0x028f: 0x050f, 0x0290: 0x0514, - 0x02aa: 0x051b, 0x02ab: 0x0520, 0x02ac: 0x0525, 0x02ad: 0x052a, 0x02ae: 0x052f, 0x02af: 0x0534, - 0x02b0: 0x0539, + 0x0281: 0x1304, 0x0282: 0x130f, 0x0283: 0x1316, 0x0284: 0x131b, + 0x0286: 0x1322, 0x0287: 0x1327, 0x0288: 0x132e, 0x0289: 0x04f6, 0x028a: 0x1333, 0x028b: 0x04fb, + 0x028c: 0x1338, 0x028d: 0x133d, 0x028e: 0x1349, 0x028f: 0x1355, 0x0290: 0x1361, 0x0291: 0x1366, + 0x0292: 0x136b, 0x0293: 0x0514, 0x0296: 0x1372, 0x0297: 0x1377, + 0x0298: 0x137e, 0x0299: 0x1383, 0x029a: 0x1388, 0x029b: 0x0500, 0x029d: 0x138d, + 0x029e: 0x1399, 0x029f: 0x13a5, 0x02a0: 0x13b1, 0x02a1: 0x13b6, 0x02a2: 0x13bb, 0x02a3: 0x0539, + 0x02a4: 0x13c2, 0x02a5: 0x13c7, 0x02a6: 0x13cc, 0x02a7: 0x13d1, 0x02a8: 0x13d8, 0x02a9: 0x13dd, + 0x02aa: 0x13e2, 0x02ab: 0x050a, 0x02ac: 0x13e7, 0x02ad: 0x13ec, 0x02ae: 0x04e3, 0x02af: 0x13f7, + 0x02b2: 0x13f9, 0x02b3: 0x1400, 0x02b4: 0x1405, + 0x02b6: 0x140c, 0x02b7: 0x1411, 0x02b8: 0x1418, 0x02b9: 0x0505, 0x02ba: 0x141d, 0x02bb: 0x050f, + 0x02bc: 0x1422, 0x02bd: 0x1427, // Block 0xb, offset 0x2c0 - 0x02ca: 0x0540, 0x02cb: 0x0545, - 0x02cc: 0x054a, 0x02cd: 0x054f, 0x02ce: 0x0554, - 0x02d3: 0x0562, 0x02d4: 0x0567, + 0x02cc: 0x1b8a, 0x02ce: 0x1b91, 0x02d0: 0x1b98, + 0x02d2: 0x1b9f, 0x02d4: 0x1ba6, 0x02d6: 0x1bad, + 0x02d8: 0x1bb4, 0x02da: 0x1bbb, 0x02dc: 0x1bc2, + 0x02de: 0x1bc9, 0x02e0: 0x1bd0, 0x02e2: 0x1bd7, + 0x02e5: 0x1bde, 0x02e7: 0x1be5, 0x02e9: 0x1bec, + 0x02f0: 0x1bf3, 0x02f1: 0x1bfa, 0x02f3: 0x1c01, 0x02f4: 0x1c08, + 0x02f6: 0x1c0f, 0x02f7: 0x1c16, 0x02f9: 0x1c1d, 0x02fa: 0x1c24, + 0x02fc: 0x1c2b, 0x02fd: 0x1c32, // Block 0xc, offset 0x300 - 0x0300: 0x0584, 0x0301: 0x0589, 0x0303: 0x058e, - 0x0307: 0x0593, - 0x030c: 0x0598, 0x030d: 0x059d, 0x030e: 0x05a2, - 0x0319: 0x05a7, - 0x0339: 0x05ac, + 0x0300: 0x2fce, 0x0301: 0x2fd2, 0x0302: 0x2fd6, 0x0303: 0x2fda, 0x0304: 0x2fde, 0x0305: 0x2fe2, + 0x0306: 0x2fe6, 0x0307: 0x2fea, 0x0308: 0x2fee, 0x0309: 0x2eed, 0x030a: 0x2ff2, 0x030b: 0x2ef1, + 0x030c: 0x2ff6, 0x030d: 0x2ffa, 0x030e: 0x2ffe, 0x030f: 0x3002, 0x0310: 0x3006, 0x0311: 0x2e6d, + 0x0312: 0x2b15, 0x0313: 0x300a, 0x0314: 0x300e, 0x0315: 0x195a, 0x0316: 0x2c25, 0x0317: 0x2d71, + 0x0318: 0x3012, 0x0319: 0x3016, 0x031a: 0x2f0d, 0x031b: 0x301a, 0x031c: 0x2f11, 0x031d: 0x301e, + 0x031e: 0x3022, 0x031f: 0x3026, 0x0320: 0x2e75, 0x0321: 0x302a, 0x0322: 0x302e, 0x0323: 0x3032, + 0x0324: 0x3036, 0x0325: 0x303a, 0x0326: 0x2e79, 0x0327: 0x303e, 0x0328: 0x3042, 0x0329: 0x3046, + 0x032a: 0x304a, 0x032b: 0x304e, 0x032c: 0x3052, 0x032d: 0x2f41, 0x032e: 0x3056, 0x032f: 0x305a, + 0x0330: 0x2cb1, 0x0331: 0x305e, 0x0332: 0x2f51, 0x0333: 0x3062, 0x0334: 0x3066, 0x0335: 0x306a, + 0x0336: 0x306e, 0x0337: 0x3072, 0x0338: 0x2f65, 0x0339: 0x3076, 0x033a: 0x2e99, 0x033b: 0x307a, + 0x033c: 0x2f69, 0x033d: 0x2bd9, 0x033e: 0x307e, 0x033f: 0x2f6d, // Block 0xd, offset 0x340 - 0x0350: 0x05b1, 0x0351: 0x05b6, - 0x0353: 0x05bb, 0x0357: 0x05c0, - 0x035c: 0x05c5, 0x035d: 0x05ca, - 0x035e: 0x05cf, - 0x0376: 0x05d4, 0x0377: 0x05d9, + 0x0340: 0x3082, 0x0341: 0x2f75, 0x0342: 0x3086, 0x0343: 0x308a, 0x0344: 0x308e, 0x0345: 0x3092, + 0x0346: 0x3096, 0x0347: 0x2f7d, 0x0348: 0x2e8d, 0x0349: 0x309a, 0x034a: 0x2f81, 0x034b: 0x309e, + 0x034c: 0x2f85, 0x034d: 0x30a2, 0x034e: 0x1b76, 0x034f: 0x30a6, 0x0350: 0x30ab, 0x0351: 0x30b0, + 0x0352: 0x30b5, 0x0353: 0x30b9, 0x0354: 0x30bd, 0x0355: 0x30c1, 0x0356: 0x30c6, 0x0357: 0x30cb, + 0x0358: 0x30d0, 0x0359: 0x30d4, // Block 0xe, offset 0x380 - 0x0381: 0x05de, 0x0382: 0x05e3, - 0x0390: 0x05e8, 0x0391: 0x05ed, - 0x0392: 0x05f2, 0x0393: 0x05f7, 0x0396: 0x05fc, 0x0397: 0x0601, - 0x039a: 0x0606, 0x039b: 0x060b, 0x039c: 0x0610, 0x039d: 0x0615, - 0x039e: 0x061a, 0x039f: 0x061f, 0x03a2: 0x0624, 0x03a3: 0x0629, - 0x03a4: 0x062e, 0x03a5: 0x0633, 0x03a6: 0x0638, 0x03a7: 0x063d, - 0x03aa: 0x0642, 0x03ab: 0x0647, 0x03ac: 0x064c, 0x03ad: 0x0651, 0x03ae: 0x0656, 0x03af: 0x065b, - 0x03b0: 0x0660, 0x03b1: 0x0665, 0x03b2: 0x066a, 0x03b3: 0x066f, 0x03b4: 0x0674, 0x03b5: 0x0679, - 0x03b8: 0x067e, 0x03b9: 0x0683, + 0x0380: 0x3d23, 0x0381: 0x3d27, 0x0382: 0x3d2b, 0x0383: 0x3d2f, 0x0384: 0x3d34, 0x0385: 0x2eb5, + 0x0386: 0x3d38, 0x0387: 0x3d3c, 0x0388: 0x3d40, 0x0389: 0x3d44, 0x038a: 0x2eb9, 0x038b: 0x3d48, + 0x038c: 0x3d4c, 0x038d: 0x3d50, 0x038e: 0x2ebd, 0x038f: 0x3d55, 0x0390: 0x3d59, 0x0391: 0x3d5d, + 0x0392: 0x3d61, 0x0393: 0x3d66, 0x0394: 0x3d6a, 0x0395: 0x3c71, 0x0396: 0x3d6e, 0x0397: 0x3d73, + 0x0398: 0x3d77, 0x0399: 0x3d7b, 0x039a: 0x3d7f, 0x039b: 0x2f9a, 0x039c: 0x3d83, 0x039d: 0x1866, + 0x039e: 0x3d88, 0x039f: 0x3d8c, 0x03a0: 0x3d90, 0x03a1: 0x3d94, 0x03a2: 0x3cb9, 0x03a3: 0x3d98, + 0x03a4: 0x3d9c, 0x03a5: 0x2fae, 0x03a6: 0x2ec1, 0x03a7: 0x2ec5, 0x03a8: 0x2fb2, 0x03a9: 0x3da0, + 0x03aa: 0x3da4, 0x03ab: 0x2bf1, 0x03ac: 0x3da8, 0x03ad: 0x2ec9, 0x03ae: 0x3dac, 0x03af: 0x3db0, + 0x03b0: 0x3db4, 0x03b1: 0x3db8, 0x03b2: 0x3db8, 0x03b3: 0x3db8, 0x03b4: 0x3dbc, 0x03b5: 0x3dc1, + 0x03b6: 0x3dc5, 0x03b7: 0x3dc9, 0x03b8: 0x3dcd, 0x03b9: 0x3dd2, 0x03ba: 0x3dd6, 0x03bb: 0x3dda, + 0x03bc: 0x3dde, 0x03bd: 0x3de2, 0x03be: 0x3de6, 0x03bf: 0x3dea, // Block 0xf, offset 0x3c0 - 0x03e2: 0x068d, 0x03e3: 0x0692, - 0x03e4: 0x0697, 0x03e5: 0x069c, 0x03e6: 0x06a1, + 0x03c0: 0x3dee, 0x03c1: 0x3df2, 0x03c2: 0x3df6, 0x03c3: 0x3dfa, 0x03c4: 0x3dfe, 0x03c5: 0x3e02, + 0x03c6: 0x3e02, 0x03c7: 0x2fba, 0x03c8: 0x3e06, 0x03c9: 0x3e0a, 0x03ca: 0x3e0e, 0x03cb: 0x3e12, + 0x03cc: 0x2ed1, 0x03cd: 0x3e16, 0x03ce: 0x3e1a, 0x03cf: 0x3e1e, 0x03d0: 0x2e39, 0x03d1: 0x3e22, + 0x03d2: 0x3e26, 0x03d3: 0x3e2a, 0x03d4: 0x3e2e, 0x03d5: 0x3e32, 0x03d6: 0x3e36, 0x03d7: 0x3e3a, + 0x03d8: 0x3e3e, 0x03d9: 0x3e42, 0x03da: 0x3e47, 0x03db: 0x3e4b, 0x03dc: 0x3e4f, 0x03dd: 0x3c55, + 0x03de: 0x3e53, 0x03df: 0x3e57, 0x03e0: 0x3e5b, 0x03e1: 0x3e60, 0x03e2: 0x3e65, 0x03e3: 0x3e69, + 0x03e4: 0x3e6d, 0x03e5: 0x3e71, 0x03e6: 0x3e75, 0x03e7: 0x3e79, 0x03e8: 0x3e7d, 0x03e9: 0x3e81, + 0x03ea: 0x3e85, 0x03eb: 0x3e85, 0x03ec: 0x3e89, 0x03ed: 0x3e8e, 0x03ee: 0x3e92, 0x03ef: 0x2be1, + 0x03f0: 0x3e96, 0x03f1: 0x3e9a, 0x03f2: 0x3e9f, 0x03f3: 0x3ea3, 0x03f4: 0x3ea7, 0x03f5: 0x18ce, + 0x03f6: 0x3eab, 0x03f7: 0x3eaf, 0x03f8: 0x18d6, 0x03f9: 0x3eb3, 0x03fa: 0x3eb7, 0x03fb: 0x3ebb, + 0x03fc: 0x3ec0, 0x03fd: 0x3ec4, 0x03fe: 0x3ec9, 0x03ff: 0x3ecd, // Block 0x10, offset 0x400 - 0x0400: 0x06ba, 0x0402: 0x06bf, - 0x0413: 0x06c4, + 0x0400: 0x3ed1, 0x0401: 0x3ed5, 0x0402: 0x3ed9, 0x0403: 0x3edd, 0x0404: 0x3ee1, 0x0405: 0x3ee5, + 0x0406: 0x3ee9, 0x0407: 0x3eed, 0x0408: 0x3ef1, 0x0409: 0x3ef5, 0x040a: 0x3efa, 0x040b: 0x3efe, + 0x040c: 0x3f02, 0x040d: 0x3f06, 0x040e: 0x2b11, 0x040f: 0x3f0a, 0x0410: 0x18fe, 0x0411: 0x3f0f, + 0x0412: 0x3f0f, 0x0413: 0x3f14, 0x0414: 0x3f18, 0x0415: 0x3f18, 0x0416: 0x3f1c, 0x0417: 0x3f20, + 0x0418: 0x3f25, 0x0419: 0x3f2a, 0x041a: 0x3f2e, 0x041b: 0x3f32, 0x041c: 0x3f36, 0x041d: 0x3f3a, + 0x041e: 0x3f3e, 0x041f: 0x3f42, 0x0420: 0x3f46, 0x0421: 0x3f4a, 0x0422: 0x3f4e, 0x0423: 0x2ee5, + 0x0424: 0x3f52, 0x0425: 0x3f57, 0x0426: 0x3f5b, 0x0427: 0x3f5f, 0x0428: 0x2fea, 0x0429: 0x3f5f, + 0x042a: 0x3f63, 0x042b: 0x2eed, 0x042c: 0x3f67, 0x042d: 0x3f6b, 0x042e: 0x3f6f, 0x042f: 0x3f73, + 0x0430: 0x2ef1, 0x0431: 0x2aa5, 0x0432: 0x3f77, 0x0433: 0x3f7b, 0x0434: 0x3f7f, 0x0435: 0x3f83, + 0x0436: 0x3f87, 0x0437: 0x3f8b, 0x0438: 0x3f8f, 0x0439: 0x3f94, 0x043a: 0x3f98, 0x043b: 0x3f9c, + 0x043c: 0x3fa0, 0x043d: 0x3fa4, 0x043e: 0x3fa8, 0x043f: 0x3fad, // Block 0x11, offset 0x440 - 0x0469: 0x06c9, - 0x0471: 0x06d0, 0x0474: 0x06d7, + 0x0440: 0x3fb1, 0x0441: 0x3fb5, 0x0442: 0x3fb9, 0x0443: 0x3fbd, 0x0444: 0x3fc1, 0x0445: 0x3fc5, + 0x0446: 0x3fc9, 0x0447: 0x3fcd, 0x0448: 0x2ef5, 0x0449: 0x3fd1, 0x044a: 0x3fd5, 0x044b: 0x3fda, + 0x044c: 0x3fde, 0x044d: 0x3fe2, 0x044e: 0x3fe6, 0x044f: 0x2efd, 0x0450: 0x3fea, 0x0451: 0x3fee, + 0x0452: 0x3ff2, 0x0453: 0x3ff6, 0x0454: 0x3ffa, 0x0455: 0x3ffe, 0x0456: 0x4002, 0x0457: 0x4006, + 0x0458: 0x2b15, 0x0459: 0x300a, 0x045a: 0x400a, 0x045b: 0x400e, 0x045c: 0x4012, 0x045d: 0x4016, + 0x045e: 0x401b, 0x045f: 0x401f, 0x0460: 0x4023, 0x0461: 0x4027, 0x0462: 0x2f01, 0x0463: 0x402b, + 0x0464: 0x4030, 0x0465: 0x4034, 0x0466: 0x4038, 0x0467: 0x30b5, 0x0468: 0x403c, 0x0469: 0x4040, + 0x046a: 0x4044, 0x046b: 0x4048, 0x046c: 0x404c, 0x046d: 0x4051, 0x046e: 0x4055, 0x046f: 0x4059, + 0x0470: 0x405d, 0x0471: 0x4062, 0x0472: 0x4066, 0x0473: 0x406a, 0x0474: 0x406e, 0x0475: 0x2c25, + 0x0476: 0x4072, 0x0477: 0x4076, 0x0478: 0x407b, 0x0479: 0x4080, 0x047a: 0x4085, 0x047b: 0x4089, + 0x047c: 0x408e, 0x047d: 0x4092, 0x047e: 0x4096, 0x047f: 0x409a, // Block 0x12, offset 0x480 - 0x0498: 0x06de, 0x0499: 0x06e5, 0x049a: 0x06ec, 0x049b: 0x06f3, 0x049c: 0x06fa, 0x049d: 0x0701, - 0x049e: 0x0708, 0x049f: 0x070f, + 0x0480: 0x409e, 0x0481: 0x2f05, 0x0482: 0x2d71, 0x0483: 0x40a2, 0x0484: 0x40a6, 0x0485: 0x40aa, + 0x0486: 0x40ae, 0x0487: 0x40b3, 0x0488: 0x40b7, 0x0489: 0x40bb, 0x048a: 0x40bf, 0x048b: 0x3016, + 0x048c: 0x40c3, 0x048d: 0x40c7, 0x048e: 0x40cc, 0x048f: 0x40d0, 0x0490: 0x40d4, 0x0491: 0x40d9, + 0x0492: 0x40de, 0x0493: 0x40e2, 0x0494: 0x301a, 0x0495: 0x40e6, 0x0496: 0x40ea, 0x0497: 0x40ee, + 0x0498: 0x40f2, 0x0499: 0x40f6, 0x049a: 0x40fa, 0x049b: 0x40fe, 0x049c: 0x4103, 0x049d: 0x4107, + 0x049e: 0x410c, 0x049f: 0x4110, 0x04a0: 0x4115, 0x04a1: 0x3022, 0x04a2: 0x4119, 0x04a3: 0x411d, + 0x04a4: 0x4122, 0x04a5: 0x4126, 0x04a6: 0x412a, 0x04a7: 0x412f, 0x04a8: 0x4134, 0x04a9: 0x4138, + 0x04aa: 0x413c, 0x04ab: 0x4140, 0x04ac: 0x4144, 0x04ad: 0x4144, 0x04ae: 0x4148, 0x04af: 0x414c, + 0x04b0: 0x302a, 0x04b1: 0x4150, 0x04b2: 0x4154, 0x04b3: 0x4158, 0x04b4: 0x415c, 0x04b5: 0x4160, + 0x04b6: 0x4165, 0x04b7: 0x4169, 0x04b8: 0x2bed, 0x04b9: 0x416e, 0x04ba: 0x4173, 0x04bb: 0x4177, + 0x04bc: 0x417c, 0x04bd: 0x4181, 0x04be: 0x4186, 0x04bf: 0x418a, // Block 0x13, offset 0x4c0 - 0x04cb: 0x0716, - 0x04cc: 0x071d, - 0x04dc: 0x0724, 0x04dd: 0x072b, - 0x04df: 0x0732, + 0x04c0: 0x3042, 0x04c1: 0x418e, 0x04c2: 0x4193, 0x04c3: 0x4198, 0x04c4: 0x419d, 0x04c5: 0x41a2, + 0x04c6: 0x41a6, 0x04c7: 0x41a6, 0x04c8: 0x3046, 0x04c9: 0x30bd, 0x04ca: 0x41aa, 0x04cb: 0x41ae, + 0x04cc: 0x41b2, 0x04cd: 0x41b6, 0x04ce: 0x41bb, 0x04cf: 0x2b59, 0x04d0: 0x304e, 0x04d1: 0x41bf, + 0x04d2: 0x41c3, 0x04d3: 0x2f2d, 0x04d4: 0x41c8, 0x04d5: 0x41cd, 0x04d6: 0x2e89, 0x04d7: 0x41d2, + 0x04d8: 0x41d6, 0x04d9: 0x2f39, 0x04da: 0x41da, 0x04db: 0x41de, 0x04dc: 0x41e2, 0x04dd: 0x41e7, + 0x04de: 0x41e7, 0x04df: 0x41ec, 0x04e0: 0x41f0, 0x04e1: 0x41f4, 0x04e2: 0x41f9, 0x04e3: 0x41fd, + 0x04e4: 0x4201, 0x04e5: 0x4205, 0x04e6: 0x420a, 0x04e7: 0x420e, 0x04e8: 0x4212, 0x04e9: 0x4216, + 0x04ea: 0x421a, 0x04eb: 0x421e, 0x04ec: 0x4223, 0x04ed: 0x4227, 0x04ee: 0x422b, 0x04ef: 0x422f, + 0x04f0: 0x4233, 0x04f1: 0x4237, 0x04f2: 0x423b, 0x04f3: 0x4240, 0x04f4: 0x4245, 0x04f5: 0x4249, + 0x04f6: 0x424e, 0x04f7: 0x4252, 0x04f8: 0x4257, 0x04f9: 0x425b, 0x04fa: 0x2f51, 0x04fb: 0x425f, + 0x04fc: 0x4264, 0x04fd: 0x4269, 0x04fe: 0x426d, 0x04ff: 0x4272, // Block 0x14, offset 0x500 - 0x0533: 0x0739, - 0x0536: 0x0740, + 0x0500: 0x4276, 0x0501: 0x427b, 0x0502: 0x427f, 0x0503: 0x4283, 0x0504: 0x4287, 0x0505: 0x428b, + 0x0506: 0x428f, 0x0507: 0x4293, 0x0508: 0x4298, 0x0509: 0x429d, 0x050a: 0x42a2, 0x050b: 0x3f14, + 0x050c: 0x42a7, 0x050d: 0x42ab, 0x050e: 0x42af, 0x050f: 0x42b3, 0x0510: 0x42b7, 0x0511: 0x42bb, + 0x0512: 0x42bf, 0x0513: 0x42c3, 0x0514: 0x42c7, 0x0515: 0x42cb, 0x0516: 0x42cf, 0x0517: 0x42d3, + 0x0518: 0x2c31, 0x0519: 0x42d8, 0x051a: 0x42dc, 0x051b: 0x42e0, 0x051c: 0x42e4, 0x051d: 0x42e8, + 0x051e: 0x42ec, 0x051f: 0x2f5d, 0x0520: 0x42f0, 0x0521: 0x42f4, 0x0522: 0x42f8, 0x0523: 0x42fc, + 0x0524: 0x4300, 0x0525: 0x4305, 0x0526: 0x430a, 0x0527: 0x430f, 0x0528: 0x4313, 0x0529: 0x4317, + 0x052a: 0x431b, 0x052b: 0x431f, 0x052c: 0x4324, 0x052d: 0x4328, 0x052e: 0x432d, 0x052f: 0x4331, + 0x0530: 0x4335, 0x0531: 0x433a, 0x0532: 0x433f, 0x0533: 0x4343, 0x0534: 0x2b45, 0x0535: 0x4347, + 0x0536: 0x434b, 0x0537: 0x434f, 0x0538: 0x4353, 0x0539: 0x4357, 0x053a: 0x435b, 0x053b: 0x306a, + 0x053c: 0x435f, 0x053d: 0x4363, 0x053e: 0x4367, 0x053f: 0x436b, // Block 0x15, offset 0x540 - 0x0559: 0x0747, 0x055a: 0x074e, 0x055b: 0x0755, - 0x055e: 0x075c, - // Block 0x16, offset 0x580 - 0x0588: 0x0763, 0x058b: 0x076a, - 0x058c: 0x0771, - 0x059c: 0x0778, 0x059d: 0x077f, - // Block 0x17, offset 0x5c0 - 0x05d4: 0x0786, - // Block 0x18, offset 0x600 - 0x060a: 0x078d, 0x060b: 0x0794, - 0x060c: 0x079b, - // Block 0x19, offset 0x640 - 0x0648: 0x07a2, - // Block 0x1a, offset 0x680 - 0x0680: 0x07a9, - 0x0687: 0x07b0, 0x0688: 0x07b7, 0x068a: 0x07be, 0x068b: 0x07c5, - // Block 0x1b, offset 0x6c0 - 0x06ca: 0x07cf, 0x06cb: 0x07d6, - 0x06cc: 0x07dd, - // Block 0x1c, offset 0x700 - 0x071a: 0x07e4, 0x071c: 0x07eb, 0x071d: 0x07f2, - 0x071e: 0x07fc, - // Block 0x1d, offset 0x740 - 0x0743: 0x0823, - 0x074d: 0x082a, - 0x0752: 0x0831, 0x0757: 0x0838, - 0x075c: 0x083f, - 0x0769: 0x0846, - 0x0773: 0x084d, 0x0775: 0x0854, - 0x0776: 0x085b, 0x0778: 0x086c, - // Block 0x1e, offset 0x780 - 0x0781: 0x087d, - 0x0793: 0x0884, - 0x079d: 0x088b, - 0x07a2: 0x0892, - 0x07a7: 0x0899, - 0x07ac: 0x08a0, - 0x07b9: 0x08a7, - // Block 0x1f, offset 0x7c0 - 0x07e6: 0x08ae, - // Block 0x20, offset 0x800 - 0x0806: 0x08b9, 0x0808: 0x08c0, 0x080a: 0x08c7, - 0x080c: 0x08ce, 0x080e: 0x08d5, - 0x0812: 0x08dc, - 0x083b: 0x08e3, - 0x083d: 0x08ea, - // Block 0x21, offset 0x840 - 0x0840: 0x08f1, 0x0841: 0x08f8, 0x0843: 0x08ff, - // Block 0x22, offset 0x880 - 0x0880: 0x09ea, 0x0881: 0x09ee, 0x0882: 0x09f2, 0x0883: 0x09f6, 0x0884: 0x09fa, 0x0885: 0x09fe, - 0x0886: 0x0a02, 0x0887: 0x0a06, 0x0888: 0x0a0a, 0x0889: 0x0a10, 0x088a: 0x0a16, 0x088b: 0x0a1a, - 0x088c: 0x0a1e, 0x088d: 0x0a22, 0x088e: 0x0a26, 0x088f: 0x0a2a, 0x0890: 0x0a2e, 0x0891: 0x0a32, - 0x0892: 0x0a36, 0x0893: 0x0a3a, 0x0894: 0x0a3e, 0x0895: 0x0a44, 0x0896: 0x0a4a, 0x0897: 0x0a50, - 0x0898: 0x0a56, 0x0899: 0x0a5a, 0x089a: 0x0a5e, 0x089b: 0x0a62, 0x089c: 0x0a66, 0x089d: 0x0a6c, - 0x089e: 0x0a72, 0x089f: 0x0a76, 0x08a0: 0x0a7a, 0x08a1: 0x0a7e, 0x08a2: 0x0a82, 0x08a3: 0x0a86, - 0x08a4: 0x0a8a, 0x08a5: 0x0a8e, 0x08a6: 0x0a92, 0x08a7: 0x0a96, 0x08a8: 0x0a9a, 0x08a9: 0x0a9e, - 0x08aa: 0x0aa2, 0x08ab: 0x0aa6, 0x08ac: 0x0aaa, 0x08ad: 0x0aae, 0x08ae: 0x0ab2, 0x08af: 0x0ab8, - 0x08b0: 0x0abe, 0x08b1: 0x0ac2, 0x08b2: 0x0ac6, 0x08b3: 0x0aca, 0x08b4: 0x0ace, 0x08b5: 0x0ad2, - 0x08b6: 0x0ad6, 0x08b7: 0x0ada, 0x08b8: 0x0ade, 0x08b9: 0x0ae4, 0x08ba: 0x0aea, 0x08bb: 0x0aee, - 0x08bc: 0x0af2, 0x08bd: 0x0af6, 0x08be: 0x0afa, 0x08bf: 0x0afe, - // Block 0x23, offset 0x8c0 - 0x08c0: 0x0b02, 0x08c1: 0x0b06, 0x08c2: 0x0b0a, 0x08c3: 0x0b0e, 0x08c4: 0x0b12, 0x08c5: 0x0b16, - 0x08c6: 0x0b1a, 0x08c7: 0x0b1e, 0x08c8: 0x0b22, 0x08c9: 0x0b26, 0x08ca: 0x0b2a, 0x08cb: 0x0b2e, - 0x08cc: 0x0b32, 0x08cd: 0x0b38, 0x08ce: 0x0b3e, 0x08cf: 0x0b44, 0x08d0: 0x0b4a, 0x08d1: 0x0b50, - 0x08d2: 0x0b56, 0x08d3: 0x0b5c, 0x08d4: 0x0b62, 0x08d5: 0x0b66, 0x08d6: 0x0b6a, 0x08d7: 0x0b6e, - 0x08d8: 0x0b72, 0x08d9: 0x0b76, 0x08da: 0x0b7a, 0x08db: 0x0b7e, 0x08dc: 0x0b82, 0x08dd: 0x0b88, - 0x08de: 0x0b8e, 0x08df: 0x0b92, 0x08e0: 0x0b96, 0x08e1: 0x0b9a, 0x08e2: 0x0b9e, 0x08e3: 0x0ba2, - 0x08e4: 0x0ba6, 0x08e5: 0x0bac, 0x08e6: 0x0bb2, 0x08e7: 0x0bb8, 0x08e8: 0x0bbe, 0x08e9: 0x0bc4, - 0x08ea: 0x0bca, 0x08eb: 0x0bce, 0x08ec: 0x0bd2, 0x08ed: 0x0bd6, 0x08ee: 0x0bda, 0x08ef: 0x0bde, - 0x08f0: 0x0be2, 0x08f1: 0x0be6, 0x08f2: 0x0bea, 0x08f3: 0x0bee, 0x08f4: 0x0bf2, 0x08f5: 0x0bf6, - 0x08f6: 0x0bfa, 0x08f7: 0x0bfe, 0x08f8: 0x0c02, 0x08f9: 0x0c08, 0x08fa: 0x0c0e, 0x08fb: 0x0c14, - 0x08fc: 0x0c1a, 0x08fd: 0x0c1e, 0x08fe: 0x0c22, 0x08ff: 0x0c26, - // Block 0x24, offset 0x900 - 0x0900: 0x0c2a, 0x0901: 0x0c2e, 0x0902: 0x0c32, 0x0903: 0x0c36, 0x0904: 0x0c3a, 0x0905: 0x0c3e, - 0x0906: 0x0c42, 0x0907: 0x0c46, 0x0908: 0x0c4a, 0x0909: 0x0c4e, 0x090a: 0x0c52, 0x090b: 0x0c56, - 0x090c: 0x0c5a, 0x090d: 0x0c5e, 0x090e: 0x0c62, 0x090f: 0x0c66, 0x0910: 0x0c6a, 0x0911: 0x0c6e, - 0x0912: 0x0c72, 0x0913: 0x0c76, 0x0914: 0x0c7a, 0x0915: 0x0c7e, 0x0916: 0x0c82, 0x0917: 0x0c86, - 0x0918: 0x0c8a, 0x0919: 0x0c8e, 0x091b: 0x0c96, - 0x0920: 0x0c9b, 0x0921: 0x0c9f, 0x0922: 0x0ca3, 0x0923: 0x0ca7, - 0x0924: 0x0cab, 0x0925: 0x0cb1, 0x0926: 0x0cb7, 0x0927: 0x0cbd, 0x0928: 0x0cc3, 0x0929: 0x0cc9, - 0x092a: 0x0ccf, 0x092b: 0x0cd5, 0x092c: 0x0cdb, 0x092d: 0x0ce1, 0x092e: 0x0ce7, 0x092f: 0x0ced, - 0x0930: 0x0cf3, 0x0931: 0x0cf9, 0x0932: 0x0cff, 0x0933: 0x0d05, 0x0934: 0x0d0b, 0x0935: 0x0d11, - 0x0936: 0x0d17, 0x0937: 0x0d1d, 0x0938: 0x0d23, 0x0939: 0x0d27, 0x093a: 0x0d2b, 0x093b: 0x0d2f, - 0x093c: 0x0d33, 0x093d: 0x0d37, 0x093e: 0x0d3b, 0x093f: 0x0d41, - // Block 0x25, offset 0x940 - 0x0940: 0x0d47, 0x0941: 0x0d4d, 0x0942: 0x0d53, 0x0943: 0x0d59, 0x0944: 0x0d5f, 0x0945: 0x0d65, - 0x0946: 0x0d6b, 0x0947: 0x0d71, 0x0948: 0x0d77, 0x0949: 0x0d7b, 0x094a: 0x0d7f, 0x094b: 0x0d83, - 0x094c: 0x0d87, 0x094d: 0x0d8b, 0x094e: 0x0d8f, 0x094f: 0x0d93, 0x0950: 0x0d97, 0x0951: 0x0d9d, - 0x0952: 0x0da3, 0x0953: 0x0da9, 0x0954: 0x0daf, 0x0955: 0x0db5, 0x0956: 0x0dbb, 0x0957: 0x0dc1, - 0x0958: 0x0dc7, 0x0959: 0x0dcd, 0x095a: 0x0dd3, 0x095b: 0x0dd9, 0x095c: 0x0ddf, 0x095d: 0x0de5, - 0x095e: 0x0deb, 0x095f: 0x0df1, 0x0960: 0x0df7, 0x0961: 0x0dfd, 0x0962: 0x0e03, 0x0963: 0x0e09, - 0x0964: 0x0e0f, 0x0965: 0x0e13, 0x0966: 0x0e17, 0x0967: 0x0e1b, 0x0968: 0x0e1f, 0x0969: 0x0e25, - 0x096a: 0x0e2b, 0x096b: 0x0e31, 0x096c: 0x0e37, 0x096d: 0x0e3d, 0x096e: 0x0e43, 0x096f: 0x0e49, - 0x0970: 0x0e4f, 0x0971: 0x0e55, 0x0972: 0x0e5b, 0x0973: 0x0e5f, 0x0974: 0x0e63, 0x0975: 0x0e67, - 0x0976: 0x0e6b, 0x0977: 0x0e6f, 0x0978: 0x0e73, 0x0979: 0x0e77, - // Block 0x26, offset 0x980 - 0x0980: 0x0e7b, 0x0981: 0x0e80, 0x0982: 0x0e85, 0x0983: 0x0e8c, 0x0984: 0x0e93, 0x0985: 0x0e9a, - 0x0986: 0x0ea1, 0x0987: 0x0ea8, 0x0988: 0x0eaf, 0x0989: 0x0eb4, 0x098a: 0x0eb9, 0x098b: 0x0ec0, - 0x098c: 0x0ec7, 0x098d: 0x0ece, 0x098e: 0x0ed5, 0x098f: 0x0edc, 0x0990: 0x0ee3, 0x0991: 0x0ee8, - 0x0992: 0x0eed, 0x0993: 0x0ef4, 0x0994: 0x0efb, 0x0995: 0x0f02, - 0x0998: 0x0f09, 0x0999: 0x0f0e, 0x099a: 0x0f13, 0x099b: 0x0f1a, 0x099c: 0x0f21, 0x099d: 0x0f28, - 0x09a0: 0x0f2f, 0x09a1: 0x0f34, 0x09a2: 0x0f39, 0x09a3: 0x0f40, - 0x09a4: 0x0f47, 0x09a5: 0x0f4e, 0x09a6: 0x0f55, 0x09a7: 0x0f5c, 0x09a8: 0x0f63, 0x09a9: 0x0f68, - 0x09aa: 0x0f6d, 0x09ab: 0x0f74, 0x09ac: 0x0f7b, 0x09ad: 0x0f82, 0x09ae: 0x0f89, 0x09af: 0x0f90, - 0x09b0: 0x0f97, 0x09b1: 0x0f9c, 0x09b2: 0x0fa1, 0x09b3: 0x0fa8, 0x09b4: 0x0faf, 0x09b5: 0x0fb6, - 0x09b6: 0x0fbd, 0x09b7: 0x0fc4, 0x09b8: 0x0fcb, 0x09b9: 0x0fd0, 0x09ba: 0x0fd5, 0x09bb: 0x0fdc, - 0x09bc: 0x0fe3, 0x09bd: 0x0fea, 0x09be: 0x0ff1, 0x09bf: 0x0ff8, - // Block 0x27, offset 0x9c0 - 0x09c0: 0x0fff, 0x09c1: 0x1004, 0x09c2: 0x1009, 0x09c3: 0x1010, 0x09c4: 0x1017, 0x09c5: 0x101e, - 0x09c8: 0x1025, 0x09c9: 0x102a, 0x09ca: 0x102f, 0x09cb: 0x1036, - 0x09cc: 0x103d, 0x09cd: 0x1044, 0x09d0: 0x104b, 0x09d1: 0x1050, - 0x09d2: 0x1055, 0x09d3: 0x105c, 0x09d4: 0x1063, 0x09d5: 0x106a, 0x09d6: 0x1071, 0x09d7: 0x1078, - 0x09d9: 0x107f, 0x09db: 0x1084, 0x09dd: 0x108b, - 0x09df: 0x1092, 0x09e0: 0x1099, 0x09e1: 0x109e, 0x09e2: 0x10a3, 0x09e3: 0x10aa, - 0x09e4: 0x10b1, 0x09e5: 0x10b8, 0x09e6: 0x10bf, 0x09e7: 0x10c6, 0x09e8: 0x10cd, 0x09e9: 0x10d2, - 0x09ea: 0x10d7, 0x09eb: 0x10de, 0x09ec: 0x10e5, 0x09ed: 0x10ec, 0x09ee: 0x10f3, 0x09ef: 0x10fa, - 0x09f0: 0x1101, 0x09f1: 0x0525, 0x09f2: 0x1106, 0x09f3: 0x052a, 0x09f4: 0x110b, 0x09f5: 0x052f, - 0x09f6: 0x1110, 0x09f7: 0x0534, 0x09f8: 0x1115, 0x09f9: 0x054a, 0x09fa: 0x111a, 0x09fb: 0x054f, - 0x09fc: 0x111f, 0x09fd: 0x0554, - // Block 0x28, offset 0xa00 - 0x0a00: 0x1124, 0x0a01: 0x112b, 0x0a02: 0x1132, 0x0a03: 0x113b, 0x0a04: 0x1144, 0x0a05: 0x114d, - 0x0a06: 0x1156, 0x0a07: 0x115f, 0x0a08: 0x1168, 0x0a09: 0x116f, 0x0a0a: 0x1176, 0x0a0b: 0x117f, - 0x0a0c: 0x1188, 0x0a0d: 0x1191, 0x0a0e: 0x119a, 0x0a0f: 0x11a3, 0x0a10: 0x11ac, 0x0a11: 0x11b3, - 0x0a12: 0x11ba, 0x0a13: 0x11c3, 0x0a14: 0x11cc, 0x0a15: 0x11d5, 0x0a16: 0x11de, 0x0a17: 0x11e7, - 0x0a18: 0x11f0, 0x0a19: 0x11f7, 0x0a1a: 0x11fe, 0x0a1b: 0x1207, 0x0a1c: 0x1210, 0x0a1d: 0x1219, - 0x0a1e: 0x1222, 0x0a1f: 0x122b, 0x0a20: 0x1234, 0x0a21: 0x123b, 0x0a22: 0x1242, 0x0a23: 0x124b, - 0x0a24: 0x1254, 0x0a25: 0x125d, 0x0a26: 0x1266, 0x0a27: 0x126f, 0x0a28: 0x1278, 0x0a29: 0x127f, - 0x0a2a: 0x1286, 0x0a2b: 0x128f, 0x0a2c: 0x1298, 0x0a2d: 0x12a1, 0x0a2e: 0x12aa, 0x0a2f: 0x12b3, - 0x0a30: 0x12bc, 0x0a31: 0x12c1, 0x0a32: 0x12c6, 0x0a33: 0x12cd, 0x0a34: 0x12d2, - 0x0a36: 0x12d9, 0x0a37: 0x12de, 0x0a38: 0x12e5, 0x0a39: 0x12ea, 0x0a3a: 0x12ef, 0x0a3b: 0x04ee, - 0x0a3c: 0x12f4, 0x0a3e: 0x12fd, - // Block 0x29, offset 0xa40 - 0x0a41: 0x1304, 0x0a42: 0x130f, 0x0a43: 0x1316, 0x0a44: 0x131b, - 0x0a46: 0x1322, 0x0a47: 0x1327, 0x0a48: 0x132e, 0x0a49: 0x04f6, 0x0a4a: 0x1333, 0x0a4b: 0x04fb, - 0x0a4c: 0x1338, 0x0a4d: 0x133d, 0x0a4e: 0x1349, 0x0a4f: 0x1355, 0x0a50: 0x1361, 0x0a51: 0x1366, - 0x0a52: 0x136b, 0x0a53: 0x0514, 0x0a56: 0x1372, 0x0a57: 0x1377, - 0x0a58: 0x137e, 0x0a59: 0x1383, 0x0a5a: 0x1388, 0x0a5b: 0x0500, 0x0a5d: 0x138d, - 0x0a5e: 0x1399, 0x0a5f: 0x13a5, 0x0a60: 0x13b1, 0x0a61: 0x13b6, 0x0a62: 0x13bb, 0x0a63: 0x0539, - 0x0a64: 0x13c2, 0x0a65: 0x13c7, 0x0a66: 0x13cc, 0x0a67: 0x13d1, 0x0a68: 0x13d8, 0x0a69: 0x13dd, - 0x0a6a: 0x13e2, 0x0a6b: 0x050a, 0x0a6c: 0x13e7, 0x0a6d: 0x13ec, 0x0a6e: 0x04e3, 0x0a6f: 0x13f7, - 0x0a72: 0x13f9, 0x0a73: 0x1400, 0x0a74: 0x1405, - 0x0a76: 0x140c, 0x0a77: 0x1411, 0x0a78: 0x1418, 0x0a79: 0x0505, 0x0a7a: 0x141d, 0x0a7b: 0x050f, - 0x0a7c: 0x1422, 0x0a7d: 0x1427, - // Block 0x2a, offset 0xa80 - 0x0a80: 0x142e, 0x0a81: 0x1432, - // Block 0x2b, offset 0xac0 - 0x0ae6: 0x14d6, - 0x0aea: 0x091c, 0x0aeb: 0x0046, - // Block 0x2c, offset 0xb00 - 0x0b1a: 0x159f, 0x0b1b: 0x15a5, - 0x0b2e: 0x15ab, - // Block 0x2d, offset 0xb40 - 0x0b4d: 0x15b1, 0x0b4e: 0x15b7, 0x0b4f: 0x15bd, - // Block 0x2e, offset 0xb80 - 0x0b84: 0x15c3, - 0x0b89: 0x15c9, - 0x0b8c: 0x15cf, - 0x0ba4: 0x15d5, 0x0ba6: 0x15db, - // Block 0x2f, offset 0xbc0 - 0x0bc1: 0x1603, 0x0bc4: 0x1609, - 0x0bc7: 0x160f, 0x0bc9: 0x1615, - 0x0be0: 0x161b, 0x0be2: 0x161f, - 0x0bed: 0x1625, 0x0bee: 0x162b, 0x0bef: 0x162f, - 0x0bf0: 0x1633, 0x0bf1: 0x1639, 0x0bf4: 0x163f, 0x0bf5: 0x1645, - 0x0bf8: 0x164b, 0x0bf9: 0x1651, - // Block 0x30, offset 0xc00 - 0x0c00: 0x1657, 0x0c01: 0x165d, 0x0c04: 0x1663, 0x0c05: 0x1669, - 0x0c08: 0x166f, 0x0c09: 0x1675, - 0x0c2c: 0x167b, 0x0c2d: 0x1681, 0x0c2e: 0x1687, 0x0c2f: 0x168d, - // Block 0x31, offset 0xc40 - 0x0c60: 0x1693, 0x0c61: 0x1699, 0x0c62: 0x169f, 0x0c63: 0x16a5, - 0x0c6a: 0x16ab, 0x0c6b: 0x16b1, 0x0c6c: 0x16b7, 0x0c6d: 0x16bd, - // Block 0x32, offset 0xc80 - 0x0ca9: 0x16c3, - 0x0caa: 0x16c7, - // Block 0x33, offset 0xcc0 - 0x0cdc: 0x1814, - // Block 0x34, offset 0xd00 - 0x0d0c: 0x1b8a, 0x0d0e: 0x1b91, 0x0d10: 0x1b98, - 0x0d12: 0x1b9f, 0x0d14: 0x1ba6, 0x0d16: 0x1bad, - 0x0d18: 0x1bb4, 0x0d1a: 0x1bbb, 0x0d1c: 0x1bc2, - 0x0d1e: 0x1bc9, 0x0d20: 0x1bd0, 0x0d22: 0x1bd7, - 0x0d25: 0x1bde, 0x0d27: 0x1be5, 0x0d29: 0x1bec, - 0x0d30: 0x1bf3, 0x0d31: 0x1bfa, 0x0d33: 0x1c01, 0x0d34: 0x1c08, - 0x0d36: 0x1c0f, 0x0d37: 0x1c16, 0x0d39: 0x1c1d, 0x0d3a: 0x1c24, - 0x0d3c: 0x1c2b, 0x0d3d: 0x1c32, - // Block 0x35, offset 0xd40 - 0x0d54: 0x1c39, - 0x0d5e: 0x1c4a, - 0x0d6c: 0x1c58, 0x0d6e: 0x1c5f, - 0x0d70: 0x1c66, 0x0d72: 0x1c6d, 0x0d74: 0x1c74, - 0x0d76: 0x1c7b, 0x0d78: 0x1c82, 0x0d7a: 0x1c89, - 0x0d7c: 0x1c90, 0x0d7e: 0x1c97, - // Block 0x36, offset 0xd80 - 0x0d80: 0x1c9e, 0x0d82: 0x1ca5, 0x0d85: 0x1cac, - 0x0d87: 0x1cb3, 0x0d89: 0x1cba, - 0x0d90: 0x1cc1, 0x0d91: 0x1cc8, - 0x0d93: 0x1ccf, 0x0d94: 0x1cd6, 0x0d96: 0x1cdd, 0x0d97: 0x1ce4, - 0x0d99: 0x1ceb, 0x0d9a: 0x1cf2, 0x0d9c: 0x1cf9, 0x0d9d: 0x1d00, - 0x0db4: 0x1d07, - 0x0db7: 0x1d0e, 0x0db8: 0x1d15, 0x0db9: 0x1d1c, 0x0dba: 0x1d23, - 0x0dbe: 0x1d2a, - // Block 0x37, offset 0xdc0 - 0x0dc0: 0x2a81, 0x0dc1: 0x2a85, 0x0dc2: 0x1a9e, 0x0dc3: 0x2a89, 0x0dc4: 0x2a8d, 0x0dc5: 0x2a91, - 0x0dc6: 0x2a95, 0x0dc7: 0x1b76, 0x0dc8: 0x1b76, 0x0dc9: 0x2a99, 0x0dca: 0x1abe, 0x0dcb: 0x2a9d, - 0x0dcc: 0x2aa1, 0x0dcd: 0x2aa5, 0x0dce: 0x2aa9, 0x0dcf: 0x2aad, 0x0dd0: 0x2ab1, 0x0dd1: 0x2ab5, - 0x0dd2: 0x2ab9, 0x0dd3: 0x2abd, 0x0dd4: 0x2ac1, 0x0dd5: 0x2ac5, 0x0dd6: 0x2ac9, 0x0dd7: 0x2acd, - 0x0dd8: 0x2ad1, 0x0dd9: 0x2ad5, 0x0dda: 0x2ad9, 0x0ddb: 0x2add, 0x0ddc: 0x2ae1, 0x0ddd: 0x2ae5, - 0x0dde: 0x2ae9, 0x0ddf: 0x2aed, 0x0de0: 0x2af1, 0x0de1: 0x2af5, 0x0de2: 0x2af9, 0x0de3: 0x2afd, - 0x0de4: 0x2b01, 0x0de5: 0x2b05, 0x0de6: 0x2b09, 0x0de7: 0x2b0d, 0x0de8: 0x2b11, 0x0de9: 0x2b15, - 0x0dea: 0x2b19, 0x0deb: 0x2b1d, 0x0dec: 0x2b21, 0x0ded: 0x2b25, 0x0dee: 0x2b29, 0x0def: 0x2b2d, - 0x0df0: 0x2b31, 0x0df1: 0x2b35, 0x0df2: 0x2b39, 0x0df3: 0x2b3d, 0x0df4: 0x1a16, 0x0df5: 0x2b41, - 0x0df6: 0x2b45, 0x0df7: 0x2b49, 0x0df8: 0x2b4d, 0x0df9: 0x2b51, 0x0dfa: 0x2b55, 0x0dfb: 0x2b59, - 0x0dfc: 0x2b5d, 0x0dfd: 0x2b61, 0x0dfe: 0x2b65, 0x0dff: 0x2b69, - // Block 0x38, offset 0xe00 - 0x0e00: 0x1b3a, 0x0e01: 0x2b6d, 0x0e02: 0x2b71, 0x0e03: 0x2b75, 0x0e04: 0x2b79, 0x0e05: 0x2b7d, - 0x0e06: 0x2b81, 0x0e07: 0x2b85, 0x0e08: 0x2b89, 0x0e09: 0x2b8d, 0x0e0a: 0x2b91, 0x0e0b: 0x2b95, - 0x0e0c: 0x2b99, 0x0e0d: 0x2b9d, 0x0e0e: 0x2ba1, 0x0e0f: 0x2ba5, 0x0e10: 0x2ba9, 0x0e11: 0x2bad, - 0x0e12: 0x2bb1, 0x0e13: 0x2bb5, 0x0e14: 0x2bb9, 0x0e15: 0x2bbd, 0x0e16: 0x2bc1, 0x0e17: 0x2bc5, - 0x0e18: 0x2bc9, 0x0e19: 0x2bcd, 0x0e1a: 0x2bd1, 0x0e1b: 0x2bd5, 0x0e1c: 0x2ac1, 0x0e1d: 0x2bd9, - 0x0e1e: 0x2bdd, 0x0e1f: 0x2be1, 0x0e20: 0x2be5, 0x0e21: 0x2be9, 0x0e22: 0x2bed, 0x0e23: 0x2bf1, - 0x0e24: 0x2bf5, 0x0e25: 0x2bf9, 0x0e26: 0x2bfd, 0x0e27: 0x2c01, 0x0e28: 0x2c05, 0x0e29: 0x2c09, - 0x0e2a: 0x2c0d, 0x0e2b: 0x2c11, 0x0e2c: 0x2c15, 0x0e2d: 0x2c19, 0x0e2e: 0x2c1d, 0x0e2f: 0x2c21, - 0x0e30: 0x2c25, 0x0e31: 0x1aa6, 0x0e32: 0x2c29, 0x0e33: 0x2c2d, 0x0e34: 0x2c31, 0x0e35: 0x2c35, - 0x0e36: 0x2c39, 0x0e37: 0x2c3d, 0x0e38: 0x2c41, 0x0e39: 0x2c45, 0x0e3a: 0x2c49, 0x0e3b: 0x2c4d, - 0x0e3c: 0x2c51, 0x0e3d: 0x2c55, 0x0e3e: 0x2c59, 0x0e3f: 0x2c5d, - // Block 0x39, offset 0xe40 - 0x0e40: 0x2c61, 0x0e41: 0x18ba, 0x0e42: 0x2c65, 0x0e43: 0x2c69, 0x0e44: 0x2c6d, 0x0e45: 0x2c71, - 0x0e46: 0x2c75, 0x0e47: 0x2c79, 0x0e48: 0x2c7d, 0x0e49: 0x2c81, 0x0e4a: 0x186e, 0x0e4b: 0x2c85, - 0x0e4c: 0x2c89, 0x0e4d: 0x2c8d, 0x0e4e: 0x2c91, 0x0e4f: 0x2c95, 0x0e50: 0x2c99, 0x0e51: 0x2c9d, - 0x0e52: 0x2ca1, 0x0e53: 0x2ca5, 0x0e54: 0x2ca9, 0x0e55: 0x2cad, 0x0e56: 0x2cb1, 0x0e57: 0x2cb5, - 0x0e58: 0x2cb9, 0x0e59: 0x2cbd, 0x0e5a: 0x2cc1, 0x0e5b: 0x2cc5, 0x0e5c: 0x2cc9, 0x0e5d: 0x2ccd, - 0x0e5e: 0x2cd1, 0x0e5f: 0x2cd5, 0x0e60: 0x2cd9, 0x0e61: 0x2c21, 0x0e62: 0x2cdd, 0x0e63: 0x2ce1, - 0x0e64: 0x2ce5, 0x0e65: 0x2ce9, 0x0e66: 0x2ced, 0x0e67: 0x2cf1, 0x0e68: 0x2cf5, 0x0e69: 0x2cf9, - 0x0e6a: 0x2be1, 0x0e6b: 0x2cfd, 0x0e6c: 0x2d01, 0x0e6d: 0x2d05, 0x0e6e: 0x2d09, 0x0e6f: 0x2d0d, - 0x0e70: 0x2d11, 0x0e71: 0x2d15, 0x0e72: 0x2d19, 0x0e73: 0x2d1d, 0x0e74: 0x2d21, 0x0e75: 0x2d25, - 0x0e76: 0x2d29, 0x0e77: 0x2d2d, 0x0e78: 0x2d31, 0x0e79: 0x2d35, 0x0e7a: 0x2d39, 0x0e7b: 0x2d3d, - 0x0e7c: 0x2d41, 0x0e7d: 0x2d45, 0x0e7e: 0x2d49, 0x0e7f: 0x2ac1, - // Block 0x3a, offset 0xe80 - 0x0e80: 0x2d4d, 0x0e81: 0x2d51, 0x0e82: 0x2d55, 0x0e83: 0x2d59, 0x0e84: 0x1b72, 0x0e85: 0x2d5d, - 0x0e86: 0x2d61, 0x0e87: 0x2d65, 0x0e88: 0x2d69, 0x0e89: 0x2d6d, 0x0e8a: 0x2d71, 0x0e8b: 0x2d75, - 0x0e8c: 0x2d79, 0x0e8d: 0x2d7d, 0x0e8e: 0x2d81, 0x0e8f: 0x2d85, 0x0e90: 0x2d89, 0x0e91: 0x2173, - 0x0e92: 0x2d8d, 0x0e93: 0x2d91, 0x0e94: 0x2d95, 0x0e95: 0x2d99, 0x0e96: 0x2d9d, 0x0e97: 0x2da1, - 0x0e98: 0x2da5, 0x0e99: 0x2da9, 0x0e9a: 0x2dad, 0x0e9b: 0x2be9, 0x0e9c: 0x2db1, 0x0e9d: 0x2db5, - 0x0e9e: 0x2db9, 0x0e9f: 0x2dbd, 0x0ea0: 0x2dc1, 0x0ea1: 0x2dc5, 0x0ea2: 0x2dc9, 0x0ea3: 0x2dcd, - 0x0ea4: 0x2dd1, 0x0ea5: 0x2dd5, 0x0ea6: 0x2dd9, 0x0ea7: 0x2ddd, 0x0ea8: 0x2de1, 0x0ea9: 0x1aba, - 0x0eaa: 0x2de5, 0x0eab: 0x2de9, 0x0eac: 0x2ded, 0x0ead: 0x2df1, 0x0eae: 0x2df5, 0x0eaf: 0x2df9, - 0x0eb0: 0x2dfd, 0x0eb1: 0x2e01, 0x0eb2: 0x2e05, 0x0eb3: 0x2e09, 0x0eb4: 0x2e0d, 0x0eb5: 0x2e11, - 0x0eb6: 0x2e15, 0x0eb7: 0x19f6, 0x0eb8: 0x2e19, 0x0eb9: 0x2e1d, 0x0eba: 0x2e21, 0x0ebb: 0x2e25, - 0x0ebc: 0x2e29, 0x0ebd: 0x2e2d, 0x0ebe: 0x2e31, 0x0ebf: 0x2e35, - // Block 0x3b, offset 0xec0 - 0x0ec0: 0x2e39, 0x0ec1: 0x2e3d, 0x0ec2: 0x2e41, 0x0ec3: 0x2e45, 0x0ec4: 0x2e49, 0x0ec5: 0x2e4d, - 0x0ec6: 0x2e51, 0x0ec7: 0x2e55, 0x0ec8: 0x1a62, 0x0ec9: 0x2e59, 0x0eca: 0x1a6e, 0x0ecb: 0x2e5d, - 0x0ecc: 0x2e61, 0x0ecd: 0x2e65, 0x0ed0: 0x2e69, - 0x0ed2: 0x2e6d, 0x0ed5: 0x2e71, 0x0ed6: 0x2e75, 0x0ed7: 0x2e79, - 0x0ed8: 0x2e7d, 0x0ed9: 0x2e81, 0x0eda: 0x2e85, 0x0edb: 0x2e89, 0x0edc: 0x2e8d, 0x0edd: 0x2e91, - 0x0ede: 0x1a12, 0x0ee0: 0x2e95, 0x0ee2: 0x2e99, - 0x0ee5: 0x2e9d, 0x0ee6: 0x2ea1, - 0x0eea: 0x2ea5, 0x0eeb: 0x2ea9, 0x0eec: 0x2ead, 0x0eed: 0x2eb1, - 0x0ef0: 0x2eb5, 0x0ef1: 0x2eb9, 0x0ef2: 0x2ebd, 0x0ef3: 0x2ec1, 0x0ef4: 0x2ec5, 0x0ef5: 0x2ec9, - 0x0ef6: 0x2ecd, 0x0ef7: 0x2ed1, 0x0ef8: 0x2ed5, 0x0ef9: 0x2ed9, 0x0efa: 0x2edd, 0x0efb: 0x2ee1, - 0x0efc: 0x18d6, 0x0efd: 0x2ee5, 0x0efe: 0x2ee9, 0x0eff: 0x2eed, - // Block 0x3c, offset 0xf00 - 0x0f00: 0x2ef1, 0x0f01: 0x2ef5, 0x0f02: 0x2ef9, 0x0f03: 0x2efd, 0x0f04: 0x2f01, 0x0f05: 0x2f05, - 0x0f06: 0x2f09, 0x0f07: 0x2f0d, 0x0f08: 0x2f11, 0x0f09: 0x2f15, 0x0f0a: 0x2f19, 0x0f0b: 0x2f1d, - 0x0f0c: 0x2187, 0x0f0d: 0x2f21, 0x0f0e: 0x2f25, 0x0f0f: 0x2f29, 0x0f10: 0x2f2d, 0x0f11: 0x2197, - 0x0f12: 0x2f31, 0x0f13: 0x2f35, 0x0f14: 0x2f39, 0x0f15: 0x2f3d, 0x0f16: 0x2f41, 0x0f17: 0x2cb1, - 0x0f18: 0x2f45, 0x0f19: 0x2f49, 0x0f1a: 0x2f4d, 0x0f1b: 0x2f51, 0x0f1c: 0x2f55, 0x0f1d: 0x2f59, - 0x0f1e: 0x2f59, 0x0f1f: 0x2f5d, 0x0f20: 0x2f61, 0x0f21: 0x2f65, 0x0f22: 0x2f69, 0x0f23: 0x2f6d, - 0x0f24: 0x2f71, 0x0f25: 0x2f75, 0x0f26: 0x2f79, 0x0f27: 0x2e9d, 0x0f28: 0x2f7d, 0x0f29: 0x2f81, - 0x0f2a: 0x2f85, 0x0f2b: 0x2f89, 0x0f2c: 0x2f8d, 0x0f2d: 0x2f92, - 0x0f30: 0x2f96, 0x0f31: 0x2f9a, 0x0f32: 0x2f9e, 0x0f33: 0x2fa2, 0x0f34: 0x2fa6, 0x0f35: 0x2faa, - 0x0f36: 0x2fae, 0x0f37: 0x2fb2, 0x0f38: 0x2ecd, 0x0f39: 0x2fb6, 0x0f3a: 0x2fba, 0x0f3b: 0x2fbe, - 0x0f3c: 0x2e69, 0x0f3d: 0x2fc2, 0x0f3e: 0x2fc6, 0x0f3f: 0x2fca, - // Block 0x3d, offset 0xf40 - 0x0f40: 0x2fce, 0x0f41: 0x2fd2, 0x0f42: 0x2fd6, 0x0f43: 0x2fda, 0x0f44: 0x2fde, 0x0f45: 0x2fe2, - 0x0f46: 0x2fe6, 0x0f47: 0x2fea, 0x0f48: 0x2fee, 0x0f49: 0x2eed, 0x0f4a: 0x2ff2, 0x0f4b: 0x2ef1, - 0x0f4c: 0x2ff6, 0x0f4d: 0x2ffa, 0x0f4e: 0x2ffe, 0x0f4f: 0x3002, 0x0f50: 0x3006, 0x0f51: 0x2e6d, - 0x0f52: 0x2b15, 0x0f53: 0x300a, 0x0f54: 0x300e, 0x0f55: 0x195a, 0x0f56: 0x2c25, 0x0f57: 0x2d71, - 0x0f58: 0x3012, 0x0f59: 0x3016, 0x0f5a: 0x2f0d, 0x0f5b: 0x301a, 0x0f5c: 0x2f11, 0x0f5d: 0x301e, - 0x0f5e: 0x3022, 0x0f5f: 0x3026, 0x0f60: 0x2e75, 0x0f61: 0x302a, 0x0f62: 0x302e, 0x0f63: 0x3032, - 0x0f64: 0x3036, 0x0f65: 0x303a, 0x0f66: 0x2e79, 0x0f67: 0x303e, 0x0f68: 0x3042, 0x0f69: 0x3046, - 0x0f6a: 0x304a, 0x0f6b: 0x304e, 0x0f6c: 0x3052, 0x0f6d: 0x2f41, 0x0f6e: 0x3056, 0x0f6f: 0x305a, - 0x0f70: 0x2cb1, 0x0f71: 0x305e, 0x0f72: 0x2f51, 0x0f73: 0x3062, 0x0f74: 0x3066, 0x0f75: 0x306a, - 0x0f76: 0x306e, 0x0f77: 0x3072, 0x0f78: 0x2f65, 0x0f79: 0x3076, 0x0f7a: 0x2e99, 0x0f7b: 0x307a, - 0x0f7c: 0x2f69, 0x0f7d: 0x2bd9, 0x0f7e: 0x307e, 0x0f7f: 0x2f6d, - // Block 0x3e, offset 0xf80 - 0x0f80: 0x3082, 0x0f81: 0x2f75, 0x0f82: 0x3086, 0x0f83: 0x308a, 0x0f84: 0x308e, 0x0f85: 0x3092, - 0x0f86: 0x3096, 0x0f87: 0x2f7d, 0x0f88: 0x2e8d, 0x0f89: 0x309a, 0x0f8a: 0x2f81, 0x0f8b: 0x309e, - 0x0f8c: 0x2f85, 0x0f8d: 0x30a2, 0x0f8e: 0x1b76, 0x0f8f: 0x30a6, 0x0f90: 0x30ab, 0x0f91: 0x30b0, - 0x0f92: 0x30b5, 0x0f93: 0x30b9, 0x0f94: 0x30bd, 0x0f95: 0x30c1, 0x0f96: 0x30c6, 0x0f97: 0x30cb, - 0x0f98: 0x30d0, 0x0f99: 0x30d4, - // Block 0x3f, offset 0xfc0 - 0x0fdd: 0x3105, - 0x0fdf: 0x310a, - 0x0fea: 0x3124, 0x0feb: 0x3129, 0x0fec: 0x312e, 0x0fed: 0x3135, 0x0fee: 0x313c, 0x0fef: 0x3141, - 0x0ff0: 0x3146, 0x0ff1: 0x314b, 0x0ff2: 0x3150, 0x0ff3: 0x3155, 0x0ff4: 0x315a, 0x0ff5: 0x315f, - 0x0ff6: 0x3164, 0x0ff8: 0x3169, 0x0ff9: 0x316e, 0x0ffa: 0x3173, 0x0ffb: 0x3178, - 0x0ffc: 0x317d, 0x0ffe: 0x3182, - // Block 0x40, offset 0x1000 - 0x1000: 0x3187, 0x1001: 0x318c, 0x1003: 0x3191, 0x1004: 0x3196, - 0x1006: 0x319b, 0x1007: 0x31a0, 0x1008: 0x31a5, 0x1009: 0x31aa, 0x100a: 0x31af, 0x100b: 0x31b4, - 0x100c: 0x31b9, 0x100d: 0x31be, 0x100e: 0x31c3, - // Block 0x41, offset 0x1040 - 0x105a: 0x3a73, 0x105c: 0x3a7c, - 0x106b: 0x3a85, - // Block 0x42, offset 0x1080 - 0x109e: 0x3a8e, 0x109f: 0x3a97, 0x10a0: 0x3aa0, 0x10a1: 0x3aad, 0x10a2: 0x3aba, 0x10a3: 0x3ac7, - 0x10a4: 0x3ad4, - // Block 0x43, offset 0x10c0 - 0x10fb: 0x3ae1, - 0x10fc: 0x3aea, 0x10fd: 0x3af3, 0x10fe: 0x3b00, 0x10ff: 0x3b0d, - // Block 0x44, offset 0x1100 - 0x1100: 0x3b1a, - // Block 0x45, offset 0x1140 - 0x1140: 0x3d23, 0x1141: 0x3d27, 0x1142: 0x3d2b, 0x1143: 0x3d2f, 0x1144: 0x3d34, 0x1145: 0x2eb5, - 0x1146: 0x3d38, 0x1147: 0x3d3c, 0x1148: 0x3d40, 0x1149: 0x3d44, 0x114a: 0x2eb9, 0x114b: 0x3d48, - 0x114c: 0x3d4c, 0x114d: 0x3d50, 0x114e: 0x2ebd, 0x114f: 0x3d55, 0x1150: 0x3d59, 0x1151: 0x3d5d, - 0x1152: 0x3d61, 0x1153: 0x3d66, 0x1154: 0x3d6a, 0x1155: 0x3c71, 0x1156: 0x3d6e, 0x1157: 0x3d73, - 0x1158: 0x3d77, 0x1159: 0x3d7b, 0x115a: 0x3d7f, 0x115b: 0x2f9a, 0x115c: 0x3d83, 0x115d: 0x1866, - 0x115e: 0x3d88, 0x115f: 0x3d8c, 0x1160: 0x3d90, 0x1161: 0x3d94, 0x1162: 0x3cb9, 0x1163: 0x3d98, - 0x1164: 0x3d9c, 0x1165: 0x2fae, 0x1166: 0x2ec1, 0x1167: 0x2ec5, 0x1168: 0x2fb2, 0x1169: 0x3da0, - 0x116a: 0x3da4, 0x116b: 0x2bf1, 0x116c: 0x3da8, 0x116d: 0x2ec9, 0x116e: 0x3dac, 0x116f: 0x3db0, - 0x1170: 0x3db4, 0x1171: 0x3db8, 0x1172: 0x3db8, 0x1173: 0x3db8, 0x1174: 0x3dbc, 0x1175: 0x3dc1, - 0x1176: 0x3dc5, 0x1177: 0x3dc9, 0x1178: 0x3dcd, 0x1179: 0x3dd2, 0x117a: 0x3dd6, 0x117b: 0x3dda, - 0x117c: 0x3dde, 0x117d: 0x3de2, 0x117e: 0x3de6, 0x117f: 0x3dea, - // Block 0x46, offset 0x1180 - 0x1180: 0x3dee, 0x1181: 0x3df2, 0x1182: 0x3df6, 0x1183: 0x3dfa, 0x1184: 0x3dfe, 0x1185: 0x3e02, - 0x1186: 0x3e02, 0x1187: 0x2fba, 0x1188: 0x3e06, 0x1189: 0x3e0a, 0x118a: 0x3e0e, 0x118b: 0x3e12, - 0x118c: 0x2ed1, 0x118d: 0x3e16, 0x118e: 0x3e1a, 0x118f: 0x3e1e, 0x1190: 0x2e39, 0x1191: 0x3e22, - 0x1192: 0x3e26, 0x1193: 0x3e2a, 0x1194: 0x3e2e, 0x1195: 0x3e32, 0x1196: 0x3e36, 0x1197: 0x3e3a, - 0x1198: 0x3e3e, 0x1199: 0x3e42, 0x119a: 0x3e47, 0x119b: 0x3e4b, 0x119c: 0x3e4f, 0x119d: 0x3c55, - 0x119e: 0x3e53, 0x119f: 0x3e57, 0x11a0: 0x3e5b, 0x11a1: 0x3e60, 0x11a2: 0x3e65, 0x11a3: 0x3e69, - 0x11a4: 0x3e6d, 0x11a5: 0x3e71, 0x11a6: 0x3e75, 0x11a7: 0x3e79, 0x11a8: 0x3e7d, 0x11a9: 0x3e81, - 0x11aa: 0x3e85, 0x11ab: 0x3e85, 0x11ac: 0x3e89, 0x11ad: 0x3e8e, 0x11ae: 0x3e92, 0x11af: 0x2be1, - 0x11b0: 0x3e96, 0x11b1: 0x3e9a, 0x11b2: 0x3e9f, 0x11b3: 0x3ea3, 0x11b4: 0x3ea7, 0x11b5: 0x18ce, - 0x11b6: 0x3eab, 0x11b7: 0x3eaf, 0x11b8: 0x18d6, 0x11b9: 0x3eb3, 0x11ba: 0x3eb7, 0x11bb: 0x3ebb, - 0x11bc: 0x3ec0, 0x11bd: 0x3ec4, 0x11be: 0x3ec9, 0x11bf: 0x3ecd, - // Block 0x47, offset 0x11c0 - 0x11c0: 0x3ed1, 0x11c1: 0x3ed5, 0x11c2: 0x3ed9, 0x11c3: 0x3edd, 0x11c4: 0x3ee1, 0x11c5: 0x3ee5, - 0x11c6: 0x3ee9, 0x11c7: 0x3eed, 0x11c8: 0x3ef1, 0x11c9: 0x3ef5, 0x11ca: 0x3efa, 0x11cb: 0x3efe, - 0x11cc: 0x3f02, 0x11cd: 0x3f06, 0x11ce: 0x2b11, 0x11cf: 0x3f0a, 0x11d0: 0x18fe, 0x11d1: 0x3f0f, - 0x11d2: 0x3f0f, 0x11d3: 0x3f14, 0x11d4: 0x3f18, 0x11d5: 0x3f18, 0x11d6: 0x3f1c, 0x11d7: 0x3f20, - 0x11d8: 0x3f25, 0x11d9: 0x3f2a, 0x11da: 0x3f2e, 0x11db: 0x3f32, 0x11dc: 0x3f36, 0x11dd: 0x3f3a, - 0x11de: 0x3f3e, 0x11df: 0x3f42, 0x11e0: 0x3f46, 0x11e1: 0x3f4a, 0x11e2: 0x3f4e, 0x11e3: 0x2ee5, - 0x11e4: 0x3f52, 0x11e5: 0x3f57, 0x11e6: 0x3f5b, 0x11e7: 0x3f5f, 0x11e8: 0x2fea, 0x11e9: 0x3f5f, - 0x11ea: 0x3f63, 0x11eb: 0x2eed, 0x11ec: 0x3f67, 0x11ed: 0x3f6b, 0x11ee: 0x3f6f, 0x11ef: 0x3f73, - 0x11f0: 0x2ef1, 0x11f1: 0x2aa5, 0x11f2: 0x3f77, 0x11f3: 0x3f7b, 0x11f4: 0x3f7f, 0x11f5: 0x3f83, - 0x11f6: 0x3f87, 0x11f7: 0x3f8b, 0x11f8: 0x3f8f, 0x11f9: 0x3f94, 0x11fa: 0x3f98, 0x11fb: 0x3f9c, - 0x11fc: 0x3fa0, 0x11fd: 0x3fa4, 0x11fe: 0x3fa8, 0x11ff: 0x3fad, - // Block 0x48, offset 0x1200 - 0x1200: 0x3fb1, 0x1201: 0x3fb5, 0x1202: 0x3fb9, 0x1203: 0x3fbd, 0x1204: 0x3fc1, 0x1205: 0x3fc5, - 0x1206: 0x3fc9, 0x1207: 0x3fcd, 0x1208: 0x2ef5, 0x1209: 0x3fd1, 0x120a: 0x3fd5, 0x120b: 0x3fda, - 0x120c: 0x3fde, 0x120d: 0x3fe2, 0x120e: 0x3fe6, 0x120f: 0x2efd, 0x1210: 0x3fea, 0x1211: 0x3fee, - 0x1212: 0x3ff2, 0x1213: 0x3ff6, 0x1214: 0x3ffa, 0x1215: 0x3ffe, 0x1216: 0x4002, 0x1217: 0x4006, - 0x1218: 0x2b15, 0x1219: 0x300a, 0x121a: 0x400a, 0x121b: 0x400e, 0x121c: 0x4012, 0x121d: 0x4016, - 0x121e: 0x401b, 0x121f: 0x401f, 0x1220: 0x4023, 0x1221: 0x4027, 0x1222: 0x2f01, 0x1223: 0x402b, - 0x1224: 0x4030, 0x1225: 0x4034, 0x1226: 0x4038, 0x1227: 0x30b5, 0x1228: 0x403c, 0x1229: 0x4040, - 0x122a: 0x4044, 0x122b: 0x4048, 0x122c: 0x404c, 0x122d: 0x4051, 0x122e: 0x4055, 0x122f: 0x4059, - 0x1230: 0x405d, 0x1231: 0x4062, 0x1232: 0x4066, 0x1233: 0x406a, 0x1234: 0x406e, 0x1235: 0x2c25, - 0x1236: 0x4072, 0x1237: 0x4076, 0x1238: 0x407b, 0x1239: 0x4080, 0x123a: 0x4085, 0x123b: 0x4089, - 0x123c: 0x408e, 0x123d: 0x4092, 0x123e: 0x4096, 0x123f: 0x409a, - // Block 0x49, offset 0x1240 - 0x1240: 0x409e, 0x1241: 0x2f05, 0x1242: 0x2d71, 0x1243: 0x40a2, 0x1244: 0x40a6, 0x1245: 0x40aa, - 0x1246: 0x40ae, 0x1247: 0x40b3, 0x1248: 0x40b7, 0x1249: 0x40bb, 0x124a: 0x40bf, 0x124b: 0x3016, - 0x124c: 0x40c3, 0x124d: 0x40c7, 0x124e: 0x40cc, 0x124f: 0x40d0, 0x1250: 0x40d4, 0x1251: 0x40d9, - 0x1252: 0x40de, 0x1253: 0x40e2, 0x1254: 0x301a, 0x1255: 0x40e6, 0x1256: 0x40ea, 0x1257: 0x40ee, - 0x1258: 0x40f2, 0x1259: 0x40f6, 0x125a: 0x40fa, 0x125b: 0x40fe, 0x125c: 0x4103, 0x125d: 0x4107, - 0x125e: 0x410c, 0x125f: 0x4110, 0x1260: 0x4115, 0x1261: 0x3022, 0x1262: 0x4119, 0x1263: 0x411d, - 0x1264: 0x4122, 0x1265: 0x4126, 0x1266: 0x412a, 0x1267: 0x412f, 0x1268: 0x4134, 0x1269: 0x4138, - 0x126a: 0x413c, 0x126b: 0x4140, 0x126c: 0x4144, 0x126d: 0x4144, 0x126e: 0x4148, 0x126f: 0x414c, - 0x1270: 0x302a, 0x1271: 0x4150, 0x1272: 0x4154, 0x1273: 0x4158, 0x1274: 0x415c, 0x1275: 0x4160, - 0x1276: 0x4165, 0x1277: 0x4169, 0x1278: 0x2bed, 0x1279: 0x416e, 0x127a: 0x4173, 0x127b: 0x4177, - 0x127c: 0x417c, 0x127d: 0x4181, 0x127e: 0x4186, 0x127f: 0x418a, - // Block 0x4a, offset 0x1280 - 0x1280: 0x3042, 0x1281: 0x418e, 0x1282: 0x4193, 0x1283: 0x4198, 0x1284: 0x419d, 0x1285: 0x41a2, - 0x1286: 0x41a6, 0x1287: 0x41a6, 0x1288: 0x3046, 0x1289: 0x30bd, 0x128a: 0x41aa, 0x128b: 0x41ae, - 0x128c: 0x41b2, 0x128d: 0x41b6, 0x128e: 0x41bb, 0x128f: 0x2b59, 0x1290: 0x304e, 0x1291: 0x41bf, - 0x1292: 0x41c3, 0x1293: 0x2f2d, 0x1294: 0x41c8, 0x1295: 0x41cd, 0x1296: 0x2e89, 0x1297: 0x41d2, - 0x1298: 0x41d6, 0x1299: 0x2f39, 0x129a: 0x41da, 0x129b: 0x41de, 0x129c: 0x41e2, 0x129d: 0x41e7, - 0x129e: 0x41e7, 0x129f: 0x41ec, 0x12a0: 0x41f0, 0x12a1: 0x41f4, 0x12a2: 0x41f9, 0x12a3: 0x41fd, - 0x12a4: 0x4201, 0x12a5: 0x4205, 0x12a6: 0x420a, 0x12a7: 0x420e, 0x12a8: 0x4212, 0x12a9: 0x4216, - 0x12aa: 0x421a, 0x12ab: 0x421e, 0x12ac: 0x4223, 0x12ad: 0x4227, 0x12ae: 0x422b, 0x12af: 0x422f, - 0x12b0: 0x4233, 0x12b1: 0x4237, 0x12b2: 0x423b, 0x12b3: 0x4240, 0x12b4: 0x4245, 0x12b5: 0x4249, - 0x12b6: 0x424e, 0x12b7: 0x4252, 0x12b8: 0x4257, 0x12b9: 0x425b, 0x12ba: 0x2f51, 0x12bb: 0x425f, - 0x12bc: 0x4264, 0x12bd: 0x4269, 0x12be: 0x426d, 0x12bf: 0x4272, - // Block 0x4b, offset 0x12c0 - 0x12c0: 0x4276, 0x12c1: 0x427b, 0x12c2: 0x427f, 0x12c3: 0x4283, 0x12c4: 0x4287, 0x12c5: 0x428b, - 0x12c6: 0x428f, 0x12c7: 0x4293, 0x12c8: 0x4298, 0x12c9: 0x429d, 0x12ca: 0x42a2, 0x12cb: 0x3f14, - 0x12cc: 0x42a7, 0x12cd: 0x42ab, 0x12ce: 0x42af, 0x12cf: 0x42b3, 0x12d0: 0x42b7, 0x12d1: 0x42bb, - 0x12d2: 0x42bf, 0x12d3: 0x42c3, 0x12d4: 0x42c7, 0x12d5: 0x42cb, 0x12d6: 0x42cf, 0x12d7: 0x42d3, - 0x12d8: 0x2c31, 0x12d9: 0x42d8, 0x12da: 0x42dc, 0x12db: 0x42e0, 0x12dc: 0x42e4, 0x12dd: 0x42e8, - 0x12de: 0x42ec, 0x12df: 0x2f5d, 0x12e0: 0x42f0, 0x12e1: 0x42f4, 0x12e2: 0x42f8, 0x12e3: 0x42fc, - 0x12e4: 0x4300, 0x12e5: 0x4305, 0x12e6: 0x430a, 0x12e7: 0x430f, 0x12e8: 0x4313, 0x12e9: 0x4317, - 0x12ea: 0x431b, 0x12eb: 0x431f, 0x12ec: 0x4324, 0x12ed: 0x4328, 0x12ee: 0x432d, 0x12ef: 0x4331, - 0x12f0: 0x4335, 0x12f1: 0x433a, 0x12f2: 0x433f, 0x12f3: 0x4343, 0x12f4: 0x2b45, 0x12f5: 0x4347, - 0x12f6: 0x434b, 0x12f7: 0x434f, 0x12f8: 0x4353, 0x12f9: 0x4357, 0x12fa: 0x435b, 0x12fb: 0x306a, - 0x12fc: 0x435f, 0x12fd: 0x4363, 0x12fe: 0x4367, 0x12ff: 0x436b, - // Block 0x4c, offset 0x1300 - 0x1300: 0x436f, 0x1301: 0x4373, 0x1302: 0x4377, 0x1303: 0x437b, 0x1304: 0x1a66, 0x1305: 0x437f, - 0x1306: 0x4384, 0x1307: 0x4388, 0x1308: 0x438c, 0x1309: 0x4390, 0x130a: 0x4394, 0x130b: 0x4398, - 0x130c: 0x439d, 0x130d: 0x43a2, 0x130e: 0x43a6, 0x130f: 0x43aa, 0x1310: 0x307e, 0x1311: 0x3082, - 0x1312: 0x1a82, 0x1313: 0x43ae, 0x1314: 0x43b3, 0x1315: 0x43b7, 0x1316: 0x43bb, 0x1317: 0x43bf, - 0x1318: 0x43c3, 0x1319: 0x43c8, 0x131a: 0x43cd, 0x131b: 0x43d1, 0x131c: 0x43d5, 0x131d: 0x43d9, - 0x131e: 0x43de, 0x131f: 0x3086, 0x1320: 0x43e2, 0x1321: 0x43e7, 0x1322: 0x43ec, 0x1323: 0x43f0, - 0x1324: 0x43f4, 0x1325: 0x43f8, 0x1326: 0x43fd, 0x1327: 0x4401, 0x1328: 0x4405, 0x1329: 0x4409, - 0x132a: 0x440d, 0x132b: 0x4411, 0x132c: 0x4415, 0x132d: 0x4419, 0x132e: 0x441e, 0x132f: 0x4422, - 0x1330: 0x4426, 0x1331: 0x442a, 0x1332: 0x442f, 0x1333: 0x4433, 0x1334: 0x4437, 0x1335: 0x443b, - 0x1336: 0x443f, 0x1337: 0x4444, 0x1338: 0x4449, 0x1339: 0x444d, 0x133a: 0x4451, 0x133b: 0x4455, - 0x133c: 0x445a, 0x133d: 0x445e, 0x133e: 0x309e, 0x133f: 0x309e, - // Block 0x4d, offset 0x1340 - 0x1340: 0x4463, 0x1341: 0x4467, 0x1342: 0x446c, 0x1343: 0x4470, 0x1344: 0x4474, 0x1345: 0x4478, - 0x1346: 0x447c, 0x1347: 0x4480, 0x1348: 0x4484, 0x1349: 0x4488, 0x134a: 0x30a2, 0x134b: 0x448d, - 0x134c: 0x4491, 0x134d: 0x4495, 0x134e: 0x4499, 0x134f: 0x449d, 0x1350: 0x44a1, 0x1351: 0x44a6, - 0x1352: 0x44aa, 0x1353: 0x44af, 0x1354: 0x44b4, 0x1355: 0x1b42, 0x1356: 0x44b9, 0x1357: 0x1b52, - 0x1358: 0x44bd, 0x1359: 0x44c1, 0x135a: 0x44c5, 0x135b: 0x44c9, 0x135c: 0x1b66, 0x135d: 0x44cd, + 0x0540: 0x436f, 0x0541: 0x4373, 0x0542: 0x4377, 0x0543: 0x437b, 0x0544: 0x1a66, 0x0545: 0x437f, + 0x0546: 0x4384, 0x0547: 0x4388, 0x0548: 0x438c, 0x0549: 0x4390, 0x054a: 0x4394, 0x054b: 0x4398, + 0x054c: 0x439d, 0x054d: 0x43a2, 0x054e: 0x43a6, 0x054f: 0x43aa, 0x0550: 0x307e, 0x0551: 0x3082, + 0x0552: 0x1a82, 0x0553: 0x43ae, 0x0554: 0x43b3, 0x0555: 0x43b7, 0x0556: 0x43bb, 0x0557: 0x43bf, + 0x0558: 0x43c3, 0x0559: 0x43c8, 0x055a: 0x43cd, 0x055b: 0x43d1, 0x055c: 0x43d5, 0x055d: 0x43d9, + 0x055e: 0x43de, 0x055f: 0x3086, 0x0560: 0x43e2, 0x0561: 0x43e7, 0x0562: 0x43ec, 0x0563: 0x43f0, + 0x0564: 0x43f4, 0x0565: 0x43f8, 0x0566: 0x43fd, 0x0567: 0x4401, 0x0568: 0x4405, 0x0569: 0x4409, + 0x056a: 0x440d, 0x056b: 0x4411, 0x056c: 0x4415, 0x056d: 0x4419, 0x056e: 0x441e, 0x056f: 0x4422, + 0x0570: 0x4426, 0x0571: 0x442a, 0x0572: 0x442f, 0x0573: 0x4433, 0x0574: 0x4437, 0x0575: 0x443b, + 0x0576: 0x443f, 0x0577: 0x4444, 0x0578: 0x4449, 0x0579: 0x444d, 0x057a: 0x4451, 0x057b: 0x4455, + 0x057c: 0x445a, 0x057d: 0x445e, 0x057e: 0x309e, 0x057f: 0x309e, +} + +// nfcDecompSparseOffset: 56 entries, 112 bytes +var nfcDecompSparseOffset = []uint16{0x0, 0xa, 0x10, 0x15, 0x18, 0x22, 0x27, 0x2e, 0x31, 0x38, 0x3e, 0x46, 0x48, 0x4c, 0x50, 0x52, 0x56, 0x59, 0x5c, 0x60, 0x62, 0x64, 0x66, 0x6a, 0x6c, 0x70, 0x7a, 0x82, 0x84, 0x8d, 0x90, 0x9e, 0xa0, 0xa4, 0xa7, 0xa9, 0xaf, 0xbb, 0xc0, 0xc3, 0xc5, 0xc7, 0xd4, 0xe2, 0xed, 0xf4, 0xff, 0x10b, 0x11c, 0x12d, 0x135, 0x139, 0x13d, 0x141, 0x145, 0x147} + +// nfcDecompSparseValues: 341 entries, 1364 bytes +var nfcDecompSparseValues = [341]valueRange{ + // Block 0x0, offset 0x1 + {value: 0x0004, lo: 0x09}, + {value: 0x0032, lo: 0x80, hi: 0x85}, + {value: 0x004a, lo: 0x87, hi: 0x8f}, + {value: 0x006e, lo: 0x91, hi: 0x96}, + {value: 0x0086, lo: 0x99, hi: 0x9d}, + {value: 0x009a, lo: 0xa0, hi: 0xa5}, + {value: 0x00b2, lo: 0xa7, hi: 0xaf}, + {value: 0x00d6, lo: 0xb1, hi: 0xb6}, + {value: 0x00ee, lo: 0xb9, hi: 0xbd}, + {value: 0x0102, lo: 0xbf, hi: 0xbf}, + // Block 0x1, offset 0x2 + {value: 0x0004, lo: 0x05}, + {value: 0x0106, lo: 0x80, hi: 0x8f}, + {value: 0x0146, lo: 0x92, hi: 0xa5}, + {value: 0x0196, lo: 0xa8, hi: 0xb0}, + {value: 0x01c0, lo: 0xb4, hi: 0xb7}, + {value: 0x01d0, lo: 0xb9, hi: 0xbe}, + // Block 0x2, offset 0x3 + {value: 0x0004, lo: 0x04}, + {value: 0x01f0, lo: 0x83, hi: 0x88}, + {value: 0x020c, lo: 0x8c, hi: 0x91}, + {value: 0x0224, lo: 0x94, hi: 0xa5}, + {value: 0x026c, lo: 0xa8, hi: 0xbe}, + // Block 0x3, offset 0x4 + {value: 0x0004, lo: 0x02}, + {value: 0x02ca, lo: 0xa0, hi: 0xa1}, + {value: 0x02d2, lo: 0xaf, hi: 0xb0}, + // Block 0x4, offset 0x5 + {value: 0x0004, lo: 0x09}, + {value: 0x03d8, lo: 0x80, hi: 0x9b}, + {value: 0x0448, lo: 0x9e, hi: 0x9f}, + {value: 0x0450, lo: 0xa6, hi: 0xaa}, + {value: 0x0466, lo: 0xab, hi: 0xab}, + {value: 0x046c, lo: 0xac, hi: 0xac}, + {value: 0x0472, lo: 0xad, hi: 0xad}, + {value: 0x0478, lo: 0xae, hi: 0xb0}, + {value: 0x0486, lo: 0xb1, hi: 0xb1}, + {value: 0x048c, lo: 0xb2, hi: 0xb3}, + // Block 0x5, offset 0x6 + {value: 0x0003, lo: 0x04}, + {value: 0x04cc, lo: 0x80, hi: 0x81}, + {value: 0x04d2, lo: 0x83, hi: 0x84}, + {value: 0x04da, lo: 0xb4, hi: 0xb4}, + {value: 0x04e1, lo: 0xbe, hi: 0xbe}, + // Block 0x6, offset 0x7 + {value: 0x0005, lo: 0x06}, + {value: 0x04e3, lo: 0x85, hi: 0x85}, + {value: 0x04ee, lo: 0x86, hi: 0x87}, + {value: 0x04f6, lo: 0x88, hi: 0x8a}, + {value: 0x0505, lo: 0x8c, hi: 0x8c}, + {value: 0x050a, lo: 0x8e, hi: 0x90}, + {value: 0x051b, lo: 0xaa, hi: 0xb0}, + // Block 0x7, offset 0x8 + {value: 0x0005, lo: 0x02}, + {value: 0x0540, lo: 0x8a, hi: 0x8e}, + {value: 0x0562, lo: 0x93, hi: 0x94}, + // Block 0x8, offset 0x9 + {value: 0x0005, lo: 0x06}, + {value: 0x0584, lo: 0x80, hi: 0x81}, + {value: 0x058e, lo: 0x83, hi: 0x83}, + {value: 0x0593, lo: 0x87, hi: 0x87}, + {value: 0x0598, lo: 0x8c, hi: 0x8e}, + {value: 0x05a7, lo: 0x99, hi: 0x99}, + {value: 0x05ac, lo: 0xb9, hi: 0xb9}, + // Block 0x9, offset 0xa + {value: 0x0005, lo: 0x05}, + {value: 0x05b1, lo: 0x90, hi: 0x91}, + {value: 0x05bb, lo: 0x93, hi: 0x93}, + {value: 0x05c0, lo: 0x97, hi: 0x97}, + {value: 0x05c5, lo: 0x9c, hi: 0x9e}, + {value: 0x05d4, lo: 0xb6, hi: 0xb7}, + // Block 0xa, offset 0xb + {value: 0x0005, lo: 0x07}, + {value: 0x05de, lo: 0x81, hi: 0x82}, + {value: 0x05e8, lo: 0x90, hi: 0x93}, + {value: 0x05fc, lo: 0x96, hi: 0x97}, + {value: 0x0606, lo: 0x9a, hi: 0x9f}, + {value: 0x0624, lo: 0xa2, hi: 0xa7}, + {value: 0x0642, lo: 0xaa, hi: 0xb5}, + {value: 0x067e, lo: 0xb8, hi: 0xb9}, + // Block 0xb, offset 0xc + {value: 0x0005, lo: 0x01}, + {value: 0x068d, lo: 0xa2, hi: 0xa6}, + // Block 0xc, offset 0xd + {value: 0x0005, lo: 0x03}, + {value: 0x06ba, lo: 0x80, hi: 0x80}, + {value: 0x06bf, lo: 0x82, hi: 0x82}, + {value: 0x06c4, lo: 0x93, hi: 0x93}, + // Block 0xd, offset 0xe + {value: 0x0007, lo: 0x03}, + {value: 0x06c9, lo: 0xa9, hi: 0xa9}, + {value: 0x06d0, lo: 0xb1, hi: 0xb1}, + {value: 0x06d7, lo: 0xb4, hi: 0xb4}, + // Block 0xe, offset 0xf + {value: 0x0007, lo: 0x01}, + {value: 0x06de, lo: 0x98, hi: 0x9f}, + // Block 0xf, offset 0x10 + {value: 0x0007, lo: 0x03}, + {value: 0x0716, lo: 0x8b, hi: 0x8c}, + {value: 0x0724, lo: 0x9c, hi: 0x9d}, + {value: 0x0732, lo: 0x9f, hi: 0x9f}, + // Block 0x10, offset 0x11 + {value: 0x0007, lo: 0x02}, + {value: 0x0739, lo: 0xb3, hi: 0xb3}, + {value: 0x0740, lo: 0xb6, hi: 0xb6}, + // Block 0x11, offset 0x12 + {value: 0x0007, lo: 0x02}, + {value: 0x0747, lo: 0x99, hi: 0x9b}, + {value: 0x075c, lo: 0x9e, hi: 0x9e}, + // Block 0x12, offset 0x13 + {value: 0x0007, lo: 0x03}, + {value: 0x0763, lo: 0x88, hi: 0x88}, + {value: 0x076a, lo: 0x8b, hi: 0x8c}, + {value: 0x0778, lo: 0x9c, hi: 0x9d}, + // Block 0x13, offset 0x14 + {value: 0x0000, lo: 0x01}, + {value: 0x0786, lo: 0x94, hi: 0x94}, + // Block 0x14, offset 0x15 + {value: 0x0007, lo: 0x01}, + {value: 0x078d, lo: 0x8a, hi: 0x8c}, + // Block 0x15, offset 0x16 + {value: 0x0000, lo: 0x01}, + {value: 0x07a2, lo: 0x88, hi: 0x88}, + // Block 0x16, offset 0x17 + {value: 0x0007, lo: 0x03}, + {value: 0x07a9, lo: 0x80, hi: 0x80}, + {value: 0x07b0, lo: 0x87, hi: 0x88}, + {value: 0x07be, lo: 0x8a, hi: 0x8b}, + // Block 0x17, offset 0x18 + {value: 0x0007, lo: 0x01}, + {value: 0x07cf, lo: 0x8a, hi: 0x8c}, + // Block 0x18, offset 0x19 + {value: 0x0007, lo: 0x03}, + {value: 0x07e4, lo: 0x9a, hi: 0x9a}, + {value: 0x07eb, lo: 0x9c, hi: 0x9d}, + {value: 0x07fc, lo: 0x9e, hi: 0x9e}, + // Block 0x19, offset 0x1a + {value: 0x0007, lo: 0x09}, + {value: 0x0823, lo: 0x83, hi: 0x83}, + {value: 0x082a, lo: 0x8d, hi: 0x8d}, + {value: 0x0831, lo: 0x92, hi: 0x92}, + {value: 0x0838, lo: 0x97, hi: 0x97}, + {value: 0x083f, lo: 0x9c, hi: 0x9c}, + {value: 0x0846, lo: 0xa9, hi: 0xa9}, + {value: 0x084d, lo: 0xb3, hi: 0xb3}, + {value: 0x0854, lo: 0xb5, hi: 0xb6}, + {value: 0x086c, lo: 0xb8, hi: 0xb8}, + // Block 0x1a, offset 0x1b + {value: 0x0007, lo: 0x07}, + {value: 0x087d, lo: 0x81, hi: 0x81}, + {value: 0x0884, lo: 0x93, hi: 0x93}, + {value: 0x088b, lo: 0x9d, hi: 0x9d}, + {value: 0x0892, lo: 0xa2, hi: 0xa2}, + {value: 0x0899, lo: 0xa7, hi: 0xa7}, + {value: 0x08a0, lo: 0xac, hi: 0xac}, + {value: 0x08a7, lo: 0xb9, hi: 0xb9}, + // Block 0x1b, offset 0x1c + {value: 0x0000, lo: 0x01}, + {value: 0x08ae, lo: 0xa6, hi: 0xa6}, + // Block 0x1c, offset 0x1d + {value: 0x0007, lo: 0x08}, + {value: 0x08b9, lo: 0x86, hi: 0x86}, + {value: 0x08c0, lo: 0x88, hi: 0x88}, + {value: 0x08c7, lo: 0x8a, hi: 0x8a}, + {value: 0x08ce, lo: 0x8c, hi: 0x8c}, + {value: 0x08d5, lo: 0x8e, hi: 0x8e}, + {value: 0x08dc, lo: 0x92, hi: 0x92}, + {value: 0x08e3, lo: 0xbb, hi: 0xbb}, + {value: 0x08ea, lo: 0xbd, hi: 0xbd}, + // Block 0x1d, offset 0x1e + {value: 0x0007, lo: 0x02}, + {value: 0x08f1, lo: 0x80, hi: 0x81}, + {value: 0x08ff, lo: 0x83, hi: 0x83}, + // Block 0x1e, offset 0x1f + {value: 0x0004, lo: 0x0d}, + {value: 0x09ea, lo: 0x80, hi: 0x88}, + {value: 0x0a10, lo: 0x89, hi: 0x89}, + {value: 0x0a16, lo: 0x8a, hi: 0x94}, + {value: 0x0a44, lo: 0x95, hi: 0x95}, + {value: 0x0a4a, lo: 0x96, hi: 0x96}, + {value: 0x0a50, lo: 0x97, hi: 0x97}, + {value: 0x0a56, lo: 0x98, hi: 0x9c}, + {value: 0x0a6c, lo: 0x9d, hi: 0x9d}, + {value: 0x0a72, lo: 0x9e, hi: 0xae}, + {value: 0x0ab8, lo: 0xaf, hi: 0xaf}, + {value: 0x0abe, lo: 0xb0, hi: 0xb8}, + {value: 0x0ae4, lo: 0xb9, hi: 0xb9}, + {value: 0x0aea, lo: 0xba, hi: 0xbf}, + // Block 0x1f, offset 0x20 + {value: 0x0004, lo: 0x01}, + {value: 0x142e, lo: 0x80, hi: 0x81}, + // Block 0x20, offset 0x21 + {value: 0x0000, lo: 0x03}, + {value: 0x14d6, lo: 0xa6, hi: 0xa6}, + {value: 0x091c, lo: 0xaa, hi: 0xaa}, + {value: 0x0046, lo: 0xab, hi: 0xab}, + // Block 0x21, offset 0x22 + {value: 0x0006, lo: 0x02}, + {value: 0x159f, lo: 0x9a, hi: 0x9b}, + {value: 0x15ab, lo: 0xae, hi: 0xae}, + // Block 0x22, offset 0x23 + {value: 0x0006, lo: 0x01}, + {value: 0x15b1, lo: 0x8d, hi: 0x8f}, + // Block 0x23, offset 0x24 + {value: 0x0006, lo: 0x05}, + {value: 0x15c3, lo: 0x84, hi: 0x84}, + {value: 0x15c9, lo: 0x89, hi: 0x89}, + {value: 0x15cf, lo: 0x8c, hi: 0x8c}, + {value: 0x15d5, lo: 0xa4, hi: 0xa4}, + {value: 0x15db, lo: 0xa6, hi: 0xa6}, + // Block 0x24, offset 0x25 + {value: 0x0006, lo: 0x0b}, + {value: 0x1603, lo: 0x81, hi: 0x81}, + {value: 0x1609, lo: 0x84, hi: 0x84}, + {value: 0x160f, lo: 0x87, hi: 0x87}, + {value: 0x1615, lo: 0x89, hi: 0x89}, + {value: 0x161b, lo: 0xa0, hi: 0xa0}, + {value: 0x161f, lo: 0xa2, hi: 0xa2}, + {value: 0x1625, lo: 0xad, hi: 0xae}, + {value: 0x162f, lo: 0xaf, hi: 0xaf}, + {value: 0x1633, lo: 0xb0, hi: 0xb1}, + {value: 0x163f, lo: 0xb4, hi: 0xb5}, + {value: 0x164b, lo: 0xb8, hi: 0xb9}, + // Block 0x25, offset 0x26 + {value: 0x0006, lo: 0x04}, + {value: 0x1657, lo: 0x80, hi: 0x81}, + {value: 0x1663, lo: 0x84, hi: 0x85}, + {value: 0x166f, lo: 0x88, hi: 0x89}, + {value: 0x167b, lo: 0xac, hi: 0xaf}, + // Block 0x26, offset 0x27 + {value: 0x0006, lo: 0x02}, + {value: 0x1693, lo: 0xa0, hi: 0xa3}, + {value: 0x16ab, lo: 0xaa, hi: 0xad}, + // Block 0x27, offset 0x28 + {value: 0x0004, lo: 0x01}, + {value: 0x16c3, lo: 0xa9, hi: 0xaa}, + // Block 0x28, offset 0x29 + {value: 0x0000, lo: 0x01}, + {value: 0x1814, lo: 0x9c, hi: 0x9c}, + // Block 0x29, offset 0x2a + {value: 0x0007, lo: 0x0c}, + {value: 0x1c39, lo: 0x94, hi: 0x94}, + {value: 0x1c4a, lo: 0x9e, hi: 0x9e}, + {value: 0x1c58, lo: 0xac, hi: 0xac}, + {value: 0x1c5f, lo: 0xae, hi: 0xae}, + {value: 0x1c66, lo: 0xb0, hi: 0xb0}, + {value: 0x1c6d, lo: 0xb2, hi: 0xb2}, + {value: 0x1c74, lo: 0xb4, hi: 0xb4}, + {value: 0x1c7b, lo: 0xb6, hi: 0xb6}, + {value: 0x1c82, lo: 0xb8, hi: 0xb8}, + {value: 0x1c89, lo: 0xba, hi: 0xba}, + {value: 0x1c90, lo: 0xbc, hi: 0xbc}, + {value: 0x1c97, lo: 0xbe, hi: 0xbe}, + // Block 0x2a, offset 0x2b + {value: 0x0007, lo: 0x0d}, + {value: 0x1c9e, lo: 0x80, hi: 0x80}, + {value: 0x1ca5, lo: 0x82, hi: 0x82}, + {value: 0x1cac, lo: 0x85, hi: 0x85}, + {value: 0x1cb3, lo: 0x87, hi: 0x87}, + {value: 0x1cba, lo: 0x89, hi: 0x89}, + {value: 0x1cc1, lo: 0x90, hi: 0x91}, + {value: 0x1ccf, lo: 0x93, hi: 0x94}, + {value: 0x1cdd, lo: 0x96, hi: 0x97}, + {value: 0x1ceb, lo: 0x99, hi: 0x9a}, + {value: 0x1cf9, lo: 0x9c, hi: 0x9d}, + {value: 0x1d07, lo: 0xb4, hi: 0xb4}, + {value: 0x1d0e, lo: 0xb7, hi: 0xba}, + {value: 0x1d2a, lo: 0xbe, hi: 0xbe}, + // Block 0x2b, offset 0x2c + {value: 0x0004, lo: 0x0a}, + {value: 0x2a81, lo: 0x80, hi: 0x81}, + {value: 0x1a9e, lo: 0x82, hi: 0x82}, + {value: 0x2a89, lo: 0x83, hi: 0x86}, + {value: 0x1b76, lo: 0x87, hi: 0x87}, + {value: 0x1b76, lo: 0x88, hi: 0x88}, + {value: 0x2a99, lo: 0x89, hi: 0x89}, + {value: 0x1abe, lo: 0x8a, hi: 0x8a}, + {value: 0x2a9d, lo: 0x8b, hi: 0xb3}, + {value: 0x1a16, lo: 0xb4, hi: 0xb4}, + {value: 0x2b41, lo: 0xb5, hi: 0xbf}, + // Block 0x2c, offset 0x2d + {value: 0x0004, lo: 0x06}, + {value: 0x1b3a, lo: 0x80, hi: 0x80}, + {value: 0x2b6d, lo: 0x81, hi: 0x9b}, + {value: 0x2ac1, lo: 0x9c, hi: 0x9c}, + {value: 0x2bd9, lo: 0x9d, hi: 0xb0}, + {value: 0x1aa6, lo: 0xb1, hi: 0xb1}, + {value: 0x2c29, lo: 0xb2, hi: 0xbf}, + // Block 0x2d, offset 0x2e + {value: 0x0004, lo: 0x0a}, + {value: 0x2c61, lo: 0x80, hi: 0x80}, + {value: 0x18ba, lo: 0x81, hi: 0x81}, + {value: 0x2c65, lo: 0x82, hi: 0x89}, + {value: 0x186e, lo: 0x8a, hi: 0x8a}, + {value: 0x2c85, lo: 0x8b, hi: 0xa0}, + {value: 0x2c21, lo: 0xa1, hi: 0xa1}, + {value: 0x2cdd, lo: 0xa2, hi: 0xa9}, + {value: 0x2be1, lo: 0xaa, hi: 0xaa}, + {value: 0x2cfd, lo: 0xab, hi: 0xbe}, + {value: 0x2ac1, lo: 0xbf, hi: 0xbf}, + // Block 0x2e, offset 0x2f + {value: 0x0004, lo: 0x0b}, + {value: 0x2d4d, lo: 0x80, hi: 0x83}, + {value: 0x1b72, lo: 0x84, hi: 0x84}, + {value: 0x2d5d, lo: 0x85, hi: 0x90}, + {value: 0x2173, lo: 0x91, hi: 0x91}, + {value: 0x2d8d, lo: 0x92, hi: 0x9a}, + {value: 0x2be9, lo: 0x9b, hi: 0x9b}, + {value: 0x2db1, lo: 0x9c, hi: 0xa8}, + {value: 0x1aba, lo: 0xa9, hi: 0xa9}, + {value: 0x2de5, lo: 0xaa, hi: 0xb6}, + {value: 0x19f6, lo: 0xb7, hi: 0xb7}, + {value: 0x2e19, lo: 0xb8, hi: 0xbf}, + // Block 0x2f, offset 0x30 + {value: 0x0004, lo: 0x10}, + {value: 0x2e39, lo: 0x80, hi: 0x87}, + {value: 0x1a62, lo: 0x88, hi: 0x88}, + {value: 0x2e59, lo: 0x89, hi: 0x89}, + {value: 0x1a6e, lo: 0x8a, hi: 0x8a}, + {value: 0x2e5d, lo: 0x8b, hi: 0x8d}, + {value: 0x2e69, lo: 0x90, hi: 0x90}, + {value: 0x2e6d, lo: 0x92, hi: 0x92}, + {value: 0x2e71, lo: 0x95, hi: 0x9d}, + {value: 0x1a12, lo: 0x9e, hi: 0x9e}, + {value: 0x2e95, lo: 0xa0, hi: 0xa0}, + {value: 0x2e99, lo: 0xa2, hi: 0xa2}, + {value: 0x2e9d, lo: 0xa5, hi: 0xa6}, + {value: 0x2ea5, lo: 0xaa, hi: 0xad}, + {value: 0x2eb5, lo: 0xb0, hi: 0xbb}, + {value: 0x18d6, lo: 0xbc, hi: 0xbc}, + {value: 0x2ee5, lo: 0xbd, hi: 0xbf}, + // Block 0x30, offset 0x31 + {value: 0x0004, lo: 0x10}, + {value: 0x2ef1, lo: 0x80, hi: 0x8b}, + {value: 0x2187, lo: 0x8c, hi: 0x8c}, + {value: 0x2f21, lo: 0x8d, hi: 0x90}, + {value: 0x2197, lo: 0x91, hi: 0x91}, + {value: 0x2f31, lo: 0x92, hi: 0x96}, + {value: 0x2cb1, lo: 0x97, hi: 0x97}, + {value: 0x2f45, lo: 0x98, hi: 0x9d}, + {value: 0x2f59, lo: 0x9e, hi: 0xa6}, + {value: 0x2e9d, lo: 0xa7, hi: 0xa7}, + {value: 0x2f7d, lo: 0xa8, hi: 0xac}, + {value: 0x2f92, lo: 0xad, hi: 0xad}, + {value: 0x2f96, lo: 0xb0, hi: 0xb7}, + {value: 0x2ecd, lo: 0xb8, hi: 0xb8}, + {value: 0x2fb6, lo: 0xb9, hi: 0xbb}, + {value: 0x2e69, lo: 0xbc, hi: 0xbc}, + {value: 0x2fc2, lo: 0xbd, hi: 0xbf}, + // Block 0x31, offset 0x32 + {value: 0x0005, lo: 0x07}, + {value: 0x3105, lo: 0x9d, hi: 0x9d}, + {value: 0x310a, lo: 0x9f, hi: 0x9f}, + {value: 0x3124, lo: 0xaa, hi: 0xac}, + {value: 0x3135, lo: 0xad, hi: 0xad}, + {value: 0x313c, lo: 0xae, hi: 0xb6}, + {value: 0x3169, lo: 0xb8, hi: 0xbc}, + {value: 0x3182, lo: 0xbe, hi: 0xbe}, + // Block 0x32, offset 0x33 + {value: 0x0005, lo: 0x03}, + {value: 0x3187, lo: 0x80, hi: 0x81}, + {value: 0x3191, lo: 0x83, hi: 0x84}, + {value: 0x319b, lo: 0x86, hi: 0x8e}, + // Block 0x33, offset 0x34 + {value: 0x0009, lo: 0x03}, + {value: 0x3a73, lo: 0x9a, hi: 0x9a}, + {value: 0x3a7c, lo: 0x9c, hi: 0x9c}, + {value: 0x3a85, lo: 0xab, hi: 0xab}, + // Block 0x34, offset 0x35 + {value: 0x000d, lo: 0x03}, + {value: 0x3a8e, lo: 0x9e, hi: 0x9e}, + {value: 0x3a97, lo: 0x9f, hi: 0x9f}, + {value: 0x3aa0, lo: 0xa0, hi: 0xa4}, + // Block 0x35, offset 0x36 + {value: 0x0009, lo: 0x03}, + {value: 0x3ae1, lo: 0xbb, hi: 0xbd}, + {value: 0x3b00, lo: 0xbe, hi: 0xbe}, + {value: 0x3b0d, lo: 0xbf, hi: 0xbf}, + // Block 0x36, offset 0x37 + {value: 0x0000, lo: 0x01}, + {value: 0x3b1a, lo: 0x80, hi: 0x80}, + // Block 0x37, offset 0x38 + {value: 0x0004, lo: 0x0d}, + {value: 0x4463, lo: 0x80, hi: 0x81}, + {value: 0x446c, lo: 0x82, hi: 0x89}, + {value: 0x30a2, lo: 0x8a, hi: 0x8a}, + {value: 0x448d, lo: 0x8b, hi: 0x90}, + {value: 0x44a6, lo: 0x91, hi: 0x92}, + {value: 0x44af, lo: 0x93, hi: 0x93}, + {value: 0x44b4, lo: 0x94, hi: 0x94}, + {value: 0x1b42, lo: 0x95, hi: 0x95}, + {value: 0x44b9, lo: 0x96, hi: 0x96}, + {value: 0x1b52, lo: 0x97, hi: 0x97}, + {value: 0x44bd, lo: 0x98, hi: 0x9b}, + {value: 0x1b66, lo: 0x9c, hi: 0x9c}, + {value: 0x44cd, lo: 0x9d, hi: 0x9d}, } // nfcDecompLookup: 832 bytes @@ -3038,37 +3125,37 @@ var nfcDecompLookup = [832]uint8{ // Block 0x1, offset 0x40 // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0x0c3: 0x03, 0x0c4: 0x04, 0x0c5: 0x05, 0x0c6: 0x06, 0x0c7: 0x07, - 0x0c8: 0x08, 0x0cd: 0x09, 0x0ce: 0x0a, 0x0cf: 0x0b, - 0x0d0: 0x0c, 0x0d1: 0x0d, 0x0d3: 0x0e, - 0x0d8: 0x0f, 0x0db: 0x10, + 0x0c3: 0x16, 0x0c4: 0x17, 0x0c5: 0x18, 0x0c6: 0x19, 0x0c7: 0x03, + 0x0c8: 0x1a, 0x0cd: 0x1b, 0x0ce: 0x1c, 0x0cf: 0x1d, + 0x0d0: 0x1e, 0x0d1: 0x1f, 0x0d3: 0x20, + 0x0d8: 0x21, 0x0db: 0x22, 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, 0x0ef: 0x08, 0x0f0: 0x0c, // Block 0x4, offset 0x100 - 0x124: 0x11, 0x125: 0x12, 0x127: 0x13, - 0x128: 0x14, 0x129: 0x15, 0x12d: 0x16, 0x12e: 0x17, 0x12f: 0x18, - 0x131: 0x19, 0x133: 0x1a, 0x135: 0x1b, 0x137: 0x1c, - 0x13d: 0x1d, 0x13e: 0x1e, + 0x124: 0x23, 0x125: 0x24, 0x127: 0x25, + 0x128: 0x26, 0x129: 0x27, 0x12d: 0x28, 0x12e: 0x29, 0x12f: 0x2a, + 0x131: 0x2b, 0x133: 0x2c, 0x135: 0x2d, 0x137: 0x2e, + 0x13d: 0x2f, 0x13e: 0x30, // Block 0x5, offset 0x140 - 0x140: 0x1f, - 0x16c: 0x20, 0x16d: 0x21, - 0x178: 0x22, 0x179: 0x23, 0x17a: 0x24, 0x17b: 0x25, 0x17c: 0x26, 0x17d: 0x27, 0x17e: 0x28, 0x17f: 0x29, + 0x140: 0x31, + 0x16c: 0x32, 0x16d: 0x33, + 0x178: 0x34, 0x179: 0x04, 0x17a: 0x05, 0x17b: 0x06, 0x17c: 0x07, 0x17d: 0x08, 0x17e: 0x09, 0x17f: 0x0a, // Block 0x6, offset 0x180 - 0x180: 0x2a, 0x184: 0x2b, 0x186: 0x2c, 0x187: 0x2d, - 0x188: 0x2e, 0x189: 0x2f, 0x18a: 0x30, 0x18b: 0x31, 0x18c: 0x32, - 0x1ab: 0x33, + 0x180: 0x35, 0x184: 0x36, 0x186: 0x37, 0x187: 0x38, + 0x188: 0x39, 0x189: 0x3a, 0x18a: 0x3b, 0x18b: 0x3c, 0x18c: 0x3d, + 0x1ab: 0x3e, // Block 0x7, offset 0x1c0 - 0x1c1: 0x34, 0x1c2: 0x35, 0x1c3: 0x36, + 0x1c1: 0x0b, 0x1c2: 0x3f, 0x1c3: 0x40, // Block 0x8, offset 0x200 - 0x224: 0x37, 0x225: 0x38, 0x226: 0x39, 0x227: 0x3a, - 0x228: 0x3b, 0x229: 0x3c, 0x22a: 0x3d, 0x22b: 0x3e, 0x22c: 0x3f, 0x22d: 0x40, + 0x224: 0x41, 0x225: 0x42, 0x226: 0x43, 0x227: 0x44, + 0x228: 0x45, 0x229: 0x46, 0x22a: 0x0c, 0x22b: 0x0d, 0x22c: 0x47, 0x22d: 0x48, // Block 0x9, offset 0x240 - 0x242: 0x41, + 0x242: 0x49, // Block 0xa, offset 0x280 - 0x285: 0x42, 0x286: 0x43, 0x287: 0x44, + 0x285: 0x4a, 0x286: 0x4b, 0x287: 0x4c, // Block 0xb, offset 0x2c0 - 0x2e0: 0x45, 0x2e1: 0x46, 0x2e2: 0x47, 0x2e3: 0x48, 0x2e4: 0x49, 0x2e5: 0x4a, 0x2e6: 0x4b, 0x2e7: 0x4c, + 0x2e0: 0x0e, 0x2e1: 0x0f, 0x2e2: 0x10, 0x2e3: 0x11, 0x2e4: 0x12, 0x2e5: 0x13, 0x2e6: 0x14, 0x2e7: 0x15, 0x2e8: 0x4d, // Block 0xc, offset 0x300 0x311: 0x09, @@ -3076,1288 +3163,1442 @@ var nfcDecompLookup = [832]uint8{ 0x32f: 0x0b, } -var nfcDecompTrie = trie{nfcDecompLookup[:], nfcDecompValues[:]} +var nfcDecompTrie = trie{nfcDecompLookup[:], nfcDecompValues[:], nfcDecompSparseValues[:], nfcDecompSparseOffset[:], 22} -// nfkcDecompValues: 10176 entries, 20352 bytes +// nfkcDecompValues: 4224 entries, 8448 bytes // Block 2 is the null block. -var nfkcDecompValues = [10176]uint16{ +var nfkcDecompValues = [4224]uint16{ // Block 0x0, offset 0x0 // Block 0x1, offset 0x40 // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0x00e0: 0x0001, - 0x00e8: 0x0003, - 0x00ea: 0x0007, 0x00ef: 0x0009, - 0x00f2: 0x000d, 0x00f3: 0x000f, 0x00f4: 0x0011, 0x00f5: 0x0015, - 0x00f8: 0x0018, 0x00f9: 0x001c, 0x00fa: 0x001e, - 0x00fc: 0x0020, 0x00fd: 0x0026, 0x00fe: 0x002c, + 0x00c4: 0x02da, 0x00c5: 0x02df, + 0x00c6: 0x02e4, 0x00c7: 0x02e9, 0x00c8: 0x02ec, 0x00c9: 0x02ef, 0x00ca: 0x02f2, 0x00cb: 0x02f5, + 0x00cc: 0x02f8, 0x00cd: 0x02fb, 0x00ce: 0x02ff, 0x00cf: 0x0303, 0x00d0: 0x0307, 0x00d1: 0x030b, + 0x00d2: 0x030f, 0x00d3: 0x0313, 0x00d4: 0x0317, 0x00d5: 0x031b, 0x00d6: 0x0321, 0x00d7: 0x0327, + 0x00d8: 0x032d, 0x00d9: 0x0333, 0x00da: 0x0339, 0x00db: 0x033f, 0x00dc: 0x0345, + 0x00de: 0x034b, 0x00df: 0x0351, 0x00e0: 0x0357, 0x00e1: 0x035d, 0x00e2: 0x0363, 0x00e3: 0x0368, + 0x00e6: 0x036d, 0x00e7: 0x0371, 0x00e8: 0x0375, 0x00e9: 0x0379, + 0x00ea: 0x037d, 0x00eb: 0x0381, 0x00ec: 0x0385, 0x00ed: 0x038b, 0x00ee: 0x0391, 0x00ef: 0x0396, + 0x00f0: 0x039b, 0x00f1: 0x039f, 0x00f2: 0x03a2, 0x00f3: 0x03a5, 0x00f4: 0x03a8, 0x00f5: 0x03ac, + 0x00f8: 0x03b0, 0x00f9: 0x03b4, 0x00fa: 0x03b8, 0x00fb: 0x03be, + 0x00fc: 0x03c4, 0x00fd: 0x03c9, 0x00fe: 0x03ce, 0x00ff: 0x03d3, // Block 0x4, offset 0x100 - 0x0100: 0x0032, 0x0101: 0x0036, 0x0102: 0x003a, 0x0103: 0x003e, 0x0104: 0x0042, 0x0105: 0x0046, - 0x0107: 0x004a, 0x0108: 0x004e, 0x0109: 0x0052, 0x010a: 0x0056, 0x010b: 0x005a, - 0x010c: 0x005e, 0x010d: 0x0062, 0x010e: 0x0066, 0x010f: 0x006a, 0x0111: 0x006e, - 0x0112: 0x0072, 0x0113: 0x0076, 0x0114: 0x007a, 0x0115: 0x007e, 0x0116: 0x0082, - 0x0119: 0x0086, 0x011a: 0x008a, 0x011b: 0x008e, 0x011c: 0x0092, 0x011d: 0x0096, - 0x0120: 0x009a, 0x0121: 0x009e, 0x0122: 0x00a2, 0x0123: 0x00a6, - 0x0124: 0x00aa, 0x0125: 0x00ae, 0x0127: 0x00b2, 0x0128: 0x00b6, 0x0129: 0x00ba, - 0x012a: 0x00be, 0x012b: 0x00c2, 0x012c: 0x00c6, 0x012d: 0x00ca, 0x012e: 0x00ce, 0x012f: 0x00d2, - 0x0131: 0x00d6, 0x0132: 0x00da, 0x0133: 0x00de, 0x0134: 0x00e2, 0x0135: 0x00e6, - 0x0136: 0x00ea, 0x0139: 0x00ee, 0x013a: 0x00f2, 0x013b: 0x00f6, - 0x013c: 0x00fa, 0x013d: 0x00fe, 0x013f: 0x0102, + 0x0100: 0x092d, 0x0101: 0x092f, 0x0102: 0x0931, 0x0103: 0x0007, 0x0104: 0x0933, 0x0105: 0x0936, + 0x0106: 0x0939, 0x0107: 0x093d, 0x0108: 0x093f, 0x0109: 0x0941, 0x010a: 0x0943, 0x010b: 0x0946, + 0x010c: 0x0949, 0x010d: 0x094c, 0x010f: 0x094e, 0x0110: 0x0950, 0x0111: 0x0952, + 0x0112: 0x001e, 0x0113: 0x0955, 0x0114: 0x0958, 0x0115: 0x095c, 0x0116: 0x0960, 0x0117: 0x0962, + 0x0118: 0x0964, 0x0119: 0x0966, 0x011a: 0x096a, 0x011b: 0x096d, 0x011c: 0x096f, 0x011d: 0x0559, + 0x011e: 0x0973, 0x011f: 0x0976, 0x0120: 0x056c, 0x0121: 0x0979, 0x0122: 0x097c, 0x0123: 0x049b, + 0x0124: 0x0964, 0x0125: 0x096d, 0x0126: 0x0559, 0x0127: 0x0973, 0x0128: 0x0575, 0x0129: 0x056c, + 0x012a: 0x0979, + 0x0138: 0x097e, // Block 0x5, offset 0x140 - 0x0140: 0x0106, 0x0141: 0x010a, 0x0142: 0x010e, 0x0143: 0x0112, 0x0144: 0x0116, 0x0145: 0x011a, - 0x0146: 0x011e, 0x0147: 0x0122, 0x0148: 0x0126, 0x0149: 0x012a, 0x014a: 0x012e, 0x014b: 0x0132, - 0x014c: 0x0136, 0x014d: 0x013a, 0x014e: 0x013e, 0x014f: 0x0142, - 0x0152: 0x0146, 0x0153: 0x014a, 0x0154: 0x014e, 0x0155: 0x0152, 0x0156: 0x0156, 0x0157: 0x015a, - 0x0158: 0x015e, 0x0159: 0x0162, 0x015a: 0x0166, 0x015b: 0x016a, 0x015c: 0x016e, 0x015d: 0x0172, - 0x015e: 0x0176, 0x015f: 0x017a, 0x0160: 0x017e, 0x0161: 0x0182, 0x0162: 0x0186, 0x0163: 0x018a, - 0x0164: 0x018e, 0x0165: 0x0192, 0x0168: 0x0196, 0x0169: 0x019a, - 0x016a: 0x019e, 0x016b: 0x01a2, 0x016c: 0x01a6, 0x016d: 0x01aa, 0x016e: 0x01ae, 0x016f: 0x01b2, - 0x0170: 0x01b6, 0x0172: 0x01ba, 0x0173: 0x01bd, 0x0174: 0x01c0, 0x0175: 0x01c4, - 0x0176: 0x01c8, 0x0177: 0x01cc, 0x0179: 0x01d0, 0x017a: 0x01d4, 0x017b: 0x01d8, - 0x017c: 0x01dc, 0x017d: 0x01e0, 0x017e: 0x01e4, 0x017f: 0x01e8, + 0x0140: 0x0b02, 0x0141: 0x0b06, 0x0142: 0x0b0a, 0x0143: 0x0b0e, 0x0144: 0x0b12, 0x0145: 0x0b16, + 0x0146: 0x0b1a, 0x0147: 0x0b1e, 0x0148: 0x0b22, 0x0149: 0x0b26, 0x014a: 0x0b2a, 0x014b: 0x0b2e, + 0x014c: 0x0b32, 0x014d: 0x0b38, 0x014e: 0x0b3e, 0x014f: 0x0b44, 0x0150: 0x0b4a, 0x0151: 0x0b50, + 0x0152: 0x0b56, 0x0153: 0x0b5c, 0x0154: 0x0b62, 0x0155: 0x0b66, 0x0156: 0x0b6a, 0x0157: 0x0b6e, + 0x0158: 0x0b72, 0x0159: 0x0b76, 0x015a: 0x0b7a, 0x015b: 0x0b7e, 0x015c: 0x0b82, 0x015d: 0x0b88, + 0x015e: 0x0b8e, 0x015f: 0x0b92, 0x0160: 0x0b96, 0x0161: 0x0b9a, 0x0162: 0x0b9e, 0x0163: 0x0ba2, + 0x0164: 0x0ba6, 0x0165: 0x0bac, 0x0166: 0x0bb2, 0x0167: 0x0bb8, 0x0168: 0x0bbe, 0x0169: 0x0bc4, + 0x016a: 0x0bca, 0x016b: 0x0bce, 0x016c: 0x0bd2, 0x016d: 0x0bd6, 0x016e: 0x0bda, 0x016f: 0x0bde, + 0x0170: 0x0be2, 0x0171: 0x0be6, 0x0172: 0x0bea, 0x0173: 0x0bee, 0x0174: 0x0bf2, 0x0175: 0x0bf6, + 0x0176: 0x0bfa, 0x0177: 0x0bfe, 0x0178: 0x0c02, 0x0179: 0x0c08, 0x017a: 0x0c0e, 0x017b: 0x0c14, + 0x017c: 0x0c1a, 0x017d: 0x0c1e, 0x017e: 0x0c22, 0x017f: 0x0c26, // Block 0x6, offset 0x180 - 0x0180: 0x01ec, 0x0183: 0x01f0, 0x0184: 0x01f4, 0x0185: 0x01f8, - 0x0186: 0x01fc, 0x0187: 0x0200, 0x0188: 0x0204, 0x0189: 0x0208, - 0x018c: 0x020c, 0x018d: 0x0210, 0x018e: 0x0214, 0x018f: 0x0218, 0x0190: 0x021c, 0x0191: 0x0220, - 0x0194: 0x0224, 0x0195: 0x0228, 0x0196: 0x022c, 0x0197: 0x0230, - 0x0198: 0x0234, 0x0199: 0x0238, 0x019a: 0x023c, 0x019b: 0x0240, 0x019c: 0x0244, 0x019d: 0x0248, - 0x019e: 0x024c, 0x019f: 0x0250, 0x01a0: 0x0254, 0x01a1: 0x0258, 0x01a2: 0x025c, 0x01a3: 0x0260, - 0x01a4: 0x0264, 0x01a5: 0x0268, 0x01a8: 0x026c, 0x01a9: 0x0270, - 0x01aa: 0x0274, 0x01ab: 0x0278, 0x01ac: 0x027c, 0x01ad: 0x0280, 0x01ae: 0x0284, 0x01af: 0x0288, - 0x01b0: 0x028c, 0x01b1: 0x0290, 0x01b2: 0x0294, 0x01b3: 0x0298, 0x01b4: 0x029c, 0x01b5: 0x02a0, - 0x01b6: 0x02a4, 0x01b7: 0x02a8, 0x01b8: 0x02ac, 0x01b9: 0x02b0, 0x01ba: 0x02b4, 0x01bb: 0x02b8, - 0x01bc: 0x02bc, 0x01bd: 0x02c0, 0x01be: 0x02c4, 0x01bf: 0x02c8, + 0x0180: 0x0c2a, 0x0181: 0x0c2e, 0x0182: 0x0c32, 0x0183: 0x0c36, 0x0184: 0x0c3a, 0x0185: 0x0c3e, + 0x0186: 0x0c42, 0x0187: 0x0c46, 0x0188: 0x0c4a, 0x0189: 0x0c4e, 0x018a: 0x0c52, 0x018b: 0x0c56, + 0x018c: 0x0c5a, 0x018d: 0x0c5e, 0x018e: 0x0c62, 0x018f: 0x0c66, 0x0190: 0x0c6a, 0x0191: 0x0c6e, + 0x0192: 0x0c72, 0x0193: 0x0c76, 0x0194: 0x0c7a, 0x0195: 0x0c7e, 0x0196: 0x0c82, 0x0197: 0x0c86, + 0x0198: 0x0c8a, 0x0199: 0x0c8e, 0x019a: 0x0c92, 0x019b: 0x0b9a, + 0x01a0: 0x0c9b, 0x01a1: 0x0c9f, 0x01a2: 0x0ca3, 0x01a3: 0x0ca7, + 0x01a4: 0x0cab, 0x01a5: 0x0cb1, 0x01a6: 0x0cb7, 0x01a7: 0x0cbd, 0x01a8: 0x0cc3, 0x01a9: 0x0cc9, + 0x01aa: 0x0ccf, 0x01ab: 0x0cd5, 0x01ac: 0x0cdb, 0x01ad: 0x0ce1, 0x01ae: 0x0ce7, 0x01af: 0x0ced, + 0x01b0: 0x0cf3, 0x01b1: 0x0cf9, 0x01b2: 0x0cff, 0x01b3: 0x0d05, 0x01b4: 0x0d0b, 0x01b5: 0x0d11, + 0x01b6: 0x0d17, 0x01b7: 0x0d1d, 0x01b8: 0x0d23, 0x01b9: 0x0d27, 0x01ba: 0x0d2b, 0x01bb: 0x0d2f, + 0x01bc: 0x0d33, 0x01bd: 0x0d37, 0x01be: 0x0d3b, 0x01bf: 0x0d41, // Block 0x7, offset 0x1c0 - 0x01e0: 0x02ca, 0x01e1: 0x02ce, - 0x01ef: 0x02d2, - 0x01f0: 0x02d6, + 0x01c0: 0x0d47, 0x01c1: 0x0d4d, 0x01c2: 0x0d53, 0x01c3: 0x0d59, 0x01c4: 0x0d5f, 0x01c5: 0x0d65, + 0x01c6: 0x0d6b, 0x01c7: 0x0d71, 0x01c8: 0x0d77, 0x01c9: 0x0d7b, 0x01ca: 0x0d7f, 0x01cb: 0x0d83, + 0x01cc: 0x0d87, 0x01cd: 0x0d8b, 0x01ce: 0x0d8f, 0x01cf: 0x0d93, 0x01d0: 0x0d97, 0x01d1: 0x0d9d, + 0x01d2: 0x0da3, 0x01d3: 0x0da9, 0x01d4: 0x0daf, 0x01d5: 0x0db5, 0x01d6: 0x0dbb, 0x01d7: 0x0dc1, + 0x01d8: 0x0dc7, 0x01d9: 0x0dcd, 0x01da: 0x0dd3, 0x01db: 0x0dd9, 0x01dc: 0x0ddf, 0x01dd: 0x0de5, + 0x01de: 0x0deb, 0x01df: 0x0df1, 0x01e0: 0x0df7, 0x01e1: 0x0dfd, 0x01e2: 0x0e03, 0x01e3: 0x0e09, + 0x01e4: 0x0e0f, 0x01e5: 0x0e13, 0x01e6: 0x0e17, 0x01e7: 0x0e1b, 0x01e8: 0x0e1f, 0x01e9: 0x0e25, + 0x01ea: 0x0e2b, 0x01eb: 0x0e31, 0x01ec: 0x0e37, 0x01ed: 0x0e3d, 0x01ee: 0x0e43, 0x01ef: 0x0e49, + 0x01f0: 0x0e4f, 0x01f1: 0x0e55, 0x01f2: 0x0e5b, 0x01f3: 0x0e5f, 0x01f4: 0x0e63, 0x01f5: 0x0e67, + 0x01f6: 0x0e6b, 0x01f7: 0x0e6f, 0x01f8: 0x0e73, 0x01f9: 0x0e77, // Block 0x8, offset 0x200 - 0x0204: 0x02da, 0x0205: 0x02df, - 0x0206: 0x02e4, 0x0207: 0x02e9, 0x0208: 0x02ec, 0x0209: 0x02ef, 0x020a: 0x02f2, 0x020b: 0x02f5, - 0x020c: 0x02f8, 0x020d: 0x02fb, 0x020e: 0x02ff, 0x020f: 0x0303, 0x0210: 0x0307, 0x0211: 0x030b, - 0x0212: 0x030f, 0x0213: 0x0313, 0x0214: 0x0317, 0x0215: 0x031b, 0x0216: 0x0321, 0x0217: 0x0327, - 0x0218: 0x032d, 0x0219: 0x0333, 0x021a: 0x0339, 0x021b: 0x033f, 0x021c: 0x0345, - 0x021e: 0x034b, 0x021f: 0x0351, 0x0220: 0x0357, 0x0221: 0x035d, 0x0222: 0x0363, 0x0223: 0x0368, - 0x0226: 0x036d, 0x0227: 0x0371, 0x0228: 0x0375, 0x0229: 0x0379, - 0x022a: 0x037d, 0x022b: 0x0381, 0x022c: 0x0385, 0x022d: 0x038b, 0x022e: 0x0391, 0x022f: 0x0396, - 0x0230: 0x039b, 0x0231: 0x039f, 0x0232: 0x03a2, 0x0233: 0x03a5, 0x0234: 0x03a8, 0x0235: 0x03ac, - 0x0238: 0x03b0, 0x0239: 0x03b4, 0x023a: 0x03b8, 0x023b: 0x03be, - 0x023c: 0x03c4, 0x023d: 0x03c9, 0x023e: 0x03ce, 0x023f: 0x03d3, + 0x0200: 0x0e7b, 0x0201: 0x0e80, 0x0202: 0x0e85, 0x0203: 0x0e8c, 0x0204: 0x0e93, 0x0205: 0x0e9a, + 0x0206: 0x0ea1, 0x0207: 0x0ea8, 0x0208: 0x0eaf, 0x0209: 0x0eb4, 0x020a: 0x0eb9, 0x020b: 0x0ec0, + 0x020c: 0x0ec7, 0x020d: 0x0ece, 0x020e: 0x0ed5, 0x020f: 0x0edc, 0x0210: 0x0ee3, 0x0211: 0x0ee8, + 0x0212: 0x0eed, 0x0213: 0x0ef4, 0x0214: 0x0efb, 0x0215: 0x0f02, + 0x0218: 0x0f09, 0x0219: 0x0f0e, 0x021a: 0x0f13, 0x021b: 0x0f1a, 0x021c: 0x0f21, 0x021d: 0x0f28, + 0x0220: 0x0f2f, 0x0221: 0x0f34, 0x0222: 0x0f39, 0x0223: 0x0f40, + 0x0224: 0x0f47, 0x0225: 0x0f4e, 0x0226: 0x0f55, 0x0227: 0x0f5c, 0x0228: 0x0f63, 0x0229: 0x0f68, + 0x022a: 0x0f6d, 0x022b: 0x0f74, 0x022c: 0x0f7b, 0x022d: 0x0f82, 0x022e: 0x0f89, 0x022f: 0x0f90, + 0x0230: 0x0f97, 0x0231: 0x0f9c, 0x0232: 0x0fa1, 0x0233: 0x0fa8, 0x0234: 0x0faf, 0x0235: 0x0fb6, + 0x0236: 0x0fbd, 0x0237: 0x0fc4, 0x0238: 0x0fcb, 0x0239: 0x0fd0, 0x023a: 0x0fd5, 0x023b: 0x0fdc, + 0x023c: 0x0fe3, 0x023d: 0x0fea, 0x023e: 0x0ff1, 0x023f: 0x0ff8, // Block 0x9, offset 0x240 - 0x0240: 0x03d8, 0x0241: 0x03dc, 0x0242: 0x03e0, 0x0243: 0x03e4, 0x0244: 0x03e8, 0x0245: 0x03ec, - 0x0246: 0x03f0, 0x0247: 0x03f4, 0x0248: 0x03f8, 0x0249: 0x03fc, 0x024a: 0x0400, 0x024b: 0x0404, - 0x024c: 0x0408, 0x024d: 0x040c, 0x024e: 0x0410, 0x024f: 0x0414, 0x0250: 0x0418, 0x0251: 0x041c, - 0x0252: 0x0420, 0x0253: 0x0424, 0x0254: 0x0428, 0x0255: 0x042c, 0x0256: 0x0430, 0x0257: 0x0434, - 0x0258: 0x0438, 0x0259: 0x043c, 0x025a: 0x0440, 0x025b: 0x0444, - 0x025e: 0x0448, 0x025f: 0x044c, - 0x0266: 0x0450, 0x0267: 0x0454, 0x0268: 0x0458, 0x0269: 0x045c, - 0x026a: 0x0460, 0x026b: 0x0466, 0x026c: 0x046c, 0x026d: 0x0472, 0x026e: 0x0478, 0x026f: 0x047c, - 0x0270: 0x0480, 0x0271: 0x0486, 0x0272: 0x048c, 0x0273: 0x0490, + 0x0240: 0x0fff, 0x0241: 0x1004, 0x0242: 0x1009, 0x0243: 0x1010, 0x0244: 0x1017, 0x0245: 0x101e, + 0x0248: 0x1025, 0x0249: 0x102a, 0x024a: 0x102f, 0x024b: 0x1036, + 0x024c: 0x103d, 0x024d: 0x1044, 0x0250: 0x104b, 0x0251: 0x1050, + 0x0252: 0x1055, 0x0253: 0x105c, 0x0254: 0x1063, 0x0255: 0x106a, 0x0256: 0x1071, 0x0257: 0x1078, + 0x0259: 0x107f, 0x025b: 0x1084, 0x025d: 0x108b, + 0x025f: 0x1092, 0x0260: 0x1099, 0x0261: 0x109e, 0x0262: 0x10a3, 0x0263: 0x10aa, + 0x0264: 0x10b1, 0x0265: 0x10b8, 0x0266: 0x10bf, 0x0267: 0x10c6, 0x0268: 0x10cd, 0x0269: 0x10d2, + 0x026a: 0x10d7, 0x026b: 0x10de, 0x026c: 0x10e5, 0x026d: 0x10ec, 0x026e: 0x10f3, 0x026f: 0x10fa, + 0x0270: 0x1101, 0x0271: 0x0525, 0x0272: 0x1106, 0x0273: 0x052a, 0x0274: 0x110b, 0x0275: 0x052f, + 0x0276: 0x1110, 0x0277: 0x0534, 0x0278: 0x1115, 0x0279: 0x054a, 0x027a: 0x111a, 0x027b: 0x054f, + 0x027c: 0x111f, 0x027d: 0x0554, // Block 0xa, offset 0x280 - 0x02b0: 0x0494, 0x02b1: 0x0496, 0x02b2: 0x0499, 0x02b3: 0x049b, 0x02b4: 0x049d, 0x02b5: 0x04a0, - 0x02b6: 0x04a3, 0x02b7: 0x04a6, 0x02b8: 0x04a8, + 0x0280: 0x1124, 0x0281: 0x112b, 0x0282: 0x1132, 0x0283: 0x113b, 0x0284: 0x1144, 0x0285: 0x114d, + 0x0286: 0x1156, 0x0287: 0x115f, 0x0288: 0x1168, 0x0289: 0x116f, 0x028a: 0x1176, 0x028b: 0x117f, + 0x028c: 0x1188, 0x028d: 0x1191, 0x028e: 0x119a, 0x028f: 0x11a3, 0x0290: 0x11ac, 0x0291: 0x11b3, + 0x0292: 0x11ba, 0x0293: 0x11c3, 0x0294: 0x11cc, 0x0295: 0x11d5, 0x0296: 0x11de, 0x0297: 0x11e7, + 0x0298: 0x11f0, 0x0299: 0x11f7, 0x029a: 0x11fe, 0x029b: 0x1207, 0x029c: 0x1210, 0x029d: 0x1219, + 0x029e: 0x1222, 0x029f: 0x122b, 0x02a0: 0x1234, 0x02a1: 0x123b, 0x02a2: 0x1242, 0x02a3: 0x124b, + 0x02a4: 0x1254, 0x02a5: 0x125d, 0x02a6: 0x1266, 0x02a7: 0x126f, 0x02a8: 0x1278, 0x02a9: 0x127f, + 0x02aa: 0x1286, 0x02ab: 0x128f, 0x02ac: 0x1298, 0x02ad: 0x12a1, 0x02ae: 0x12aa, 0x02af: 0x12b3, + 0x02b0: 0x12bc, 0x02b1: 0x12c1, 0x02b2: 0x12c6, 0x02b3: 0x12cd, 0x02b4: 0x12d2, + 0x02b6: 0x12d9, 0x02b7: 0x12de, 0x02b8: 0x12e5, 0x02b9: 0x12ea, 0x02ba: 0x12ef, 0x02bb: 0x04ee, + 0x02bc: 0x12f4, 0x02bd: 0x12f9, 0x02be: 0x12fd, 0x02bf: 0x12f9, // Block 0xb, offset 0x2c0 - 0x02d8: 0x04aa, 0x02d9: 0x04ae, 0x02da: 0x04b2, 0x02db: 0x04b6, 0x02dc: 0x04ba, 0x02dd: 0x04be, - 0x02e0: 0x04c2, 0x02e1: 0x04c5, 0x02e2: 0x02c8, 0x02e3: 0x04c7, - 0x02e4: 0x04c9, + 0x02c0: 0x1300, 0x02c1: 0x1309, 0x02c2: 0x130f, 0x02c3: 0x1316, 0x02c4: 0x131b, + 0x02c6: 0x1322, 0x02c7: 0x1327, 0x02c8: 0x132e, 0x02c9: 0x04f6, 0x02ca: 0x1333, 0x02cb: 0x04fb, + 0x02cc: 0x1338, 0x02cd: 0x1343, 0x02ce: 0x134f, 0x02cf: 0x135b, 0x02d0: 0x1361, 0x02d1: 0x1366, + 0x02d2: 0x136b, 0x02d3: 0x0514, 0x02d6: 0x1372, 0x02d7: 0x1377, + 0x02d8: 0x137e, 0x02d9: 0x1383, 0x02da: 0x1388, 0x02db: 0x0500, 0x02dd: 0x1393, + 0x02de: 0x139f, 0x02df: 0x13ab, 0x02e0: 0x13b1, 0x02e1: 0x13b6, 0x02e2: 0x13bb, 0x02e3: 0x0539, + 0x02e4: 0x13c2, 0x02e5: 0x13c7, 0x02e6: 0x13cc, 0x02e7: 0x13d1, 0x02e8: 0x13d8, 0x02e9: 0x13dd, + 0x02ea: 0x13e2, 0x02eb: 0x050a, 0x02ec: 0x13e7, 0x02ed: 0x13f1, 0x02ee: 0x04e8, 0x02ef: 0x13f7, + 0x02f2: 0x13f9, 0x02f3: 0x1400, 0x02f4: 0x1405, + 0x02f6: 0x140c, 0x02f7: 0x1411, 0x02f8: 0x1418, 0x02f9: 0x0505, 0x02fa: 0x141d, 0x02fb: 0x050f, + 0x02fc: 0x1422, 0x02fd: 0x0011, 0x02fe: 0x142a, // Block 0xc, offset 0x300 - 0x0300: 0x04cc, 0x0301: 0x04cf, 0x0303: 0x04d2, 0x0304: 0x04d5, - 0x0334: 0x04da, - 0x033a: 0x04dd, - 0x033e: 0x04e1, + 0x0300: 0x1486, 0x0301: 0x001c, 0x0302: 0x000d, 0x0303: 0x000f, 0x0304: 0x1488, 0x0305: 0x148a, + 0x0306: 0x148c, 0x0307: 0x148e, 0x0308: 0x1490, 0x0309: 0x1492, 0x030a: 0x1494, 0x030b: 0x1496, + 0x030c: 0x149a, 0x030d: 0x149c, 0x030e: 0x149e, 0x0310: 0x0007, 0x0311: 0x0941, + 0x0312: 0x001e, 0x0313: 0x04c7, 0x0314: 0x0943, 0x0315: 0x0494, 0x0316: 0x094e, 0x0317: 0x04c5, + 0x0318: 0x0950, 0x0319: 0x14a0, 0x031a: 0x0960, 0x031b: 0x02c8, 0x031c: 0x0962, + 0x0328: 0x14a2, // Block 0xd, offset 0x340 - 0x0344: 0x0011, 0x0345: 0x04e8, - 0x0346: 0x04ee, 0x0347: 0x04f3, 0x0348: 0x04f6, 0x0349: 0x04fb, 0x034a: 0x0500, - 0x034c: 0x0505, 0x034e: 0x050a, 0x034f: 0x050f, 0x0350: 0x0514, - 0x036a: 0x051b, 0x036b: 0x0520, 0x036c: 0x0525, 0x036d: 0x052a, 0x036e: 0x052f, 0x036f: 0x0534, - 0x0370: 0x0539, + 0x0340: 0x14a5, 0x0341: 0x14a9, 0x0342: 0x14ad, 0x0343: 0x14af, 0x0345: 0x14b3, + 0x0346: 0x14b7, 0x0347: 0x14bb, 0x0349: 0x14be, 0x034a: 0x094c, 0x034b: 0x0916, + 0x034c: 0x0916, 0x034d: 0x0916, 0x034e: 0x0494, 0x034f: 0x14c2, 0x0350: 0x0918, 0x0351: 0x0918, + 0x0352: 0x091e, 0x0353: 0x04c5, 0x0355: 0x0922, 0x0356: 0x14c5, + 0x0359: 0x0929, 0x035a: 0x14c8, 0x035b: 0x092b, 0x035c: 0x092b, 0x035d: 0x092b, + 0x0360: 0x14ca, 0x0361: 0x14cd, 0x0362: 0x14d1, + 0x0364: 0x14d4, 0x0366: 0x14d6, 0x0368: 0x14d4, + 0x036a: 0x091c, 0x036b: 0x0046, 0x036c: 0x090b, 0x036d: 0x14ad, 0x036f: 0x0941, + 0x0370: 0x090f, 0x0371: 0x14d9, 0x0373: 0x0920, 0x0374: 0x001e, 0x0375: 0x14db, + 0x0376: 0x14de, 0x0377: 0x14e1, 0x0378: 0x14e4, 0x0379: 0x097c, 0x037b: 0x14e7, + 0x037c: 0x056f, 0x037d: 0x0973, 0x037e: 0x14eb, 0x037f: 0x14ee, // Block 0xe, offset 0x380 - 0x038a: 0x0540, 0x038b: 0x0545, - 0x038c: 0x054a, 0x038d: 0x054f, 0x038e: 0x0554, 0x0390: 0x0559, 0x0391: 0x055c, - 0x0392: 0x055f, 0x0393: 0x050a, 0x0394: 0x0520, 0x0395: 0x056c, 0x0396: 0x056f, - 0x03b0: 0x0572, 0x03b1: 0x0575, 0x03b2: 0x0578, 0x03b4: 0x057b, 0x03b5: 0x057e, - 0x03b9: 0x0581, + 0x0380: 0x14f1, 0x0385: 0x090d, + 0x0386: 0x093f, 0x0387: 0x0941, 0x0388: 0x097c, 0x0389: 0x0499, + 0x0390: 0x14f5, 0x0391: 0x14fb, + 0x0392: 0x1501, 0x0393: 0x1508, 0x0394: 0x150e, 0x0395: 0x1514, 0x0396: 0x151a, 0x0397: 0x1520, + 0x0398: 0x1526, 0x0399: 0x152c, 0x039a: 0x1532, 0x039b: 0x1538, 0x039c: 0x153e, 0x039d: 0x1544, + 0x039e: 0x154a, 0x039f: 0x1550, 0x03a0: 0x0918, 0x03a1: 0x1555, 0x03a2: 0x1558, 0x03a3: 0x155c, + 0x03a4: 0x155f, 0x03a5: 0x1561, 0x03a6: 0x1564, 0x03a7: 0x1568, 0x03a8: 0x156d, 0x03a9: 0x1570, + 0x03aa: 0x1572, 0x03ab: 0x1575, 0x03ac: 0x091e, 0x03ad: 0x14ad, 0x03ae: 0x090d, 0x03af: 0x0920, + 0x03b0: 0x097c, 0x03b1: 0x1579, 0x03b2: 0x157c, 0x03b3: 0x1580, 0x03b4: 0x096d, 0x03b5: 0x1583, + 0x03b6: 0x1586, 0x03b7: 0x158a, 0x03b8: 0x158f, 0x03b9: 0x04c7, 0x03ba: 0x1592, 0x03bb: 0x1595, + 0x03bc: 0x04c5, 0x03bd: 0x0984, 0x03be: 0x093f, 0x03bf: 0x0950, // Block 0xf, offset 0x3c0 - 0x03c0: 0x0584, 0x03c1: 0x0589, 0x03c3: 0x058e, - 0x03c7: 0x0593, - 0x03cc: 0x0598, 0x03cd: 0x059d, 0x03ce: 0x05a2, - 0x03d9: 0x05a7, - 0x03f9: 0x05ac, + 0x03e0: 0x001c, 0x03e1: 0x000d, 0x03e2: 0x000f, 0x03e3: 0x1488, + 0x03e4: 0x148a, 0x03e5: 0x148c, 0x03e6: 0x148e, 0x03e7: 0x1490, 0x03e8: 0x1492, 0x03e9: 0x16cb, + 0x03ea: 0x16ce, 0x03eb: 0x16d1, 0x03ec: 0x16d4, 0x03ed: 0x16d7, 0x03ee: 0x16da, 0x03ef: 0x16dd, + 0x03f0: 0x16e0, 0x03f1: 0x16e3, 0x03f2: 0x16e6, 0x03f3: 0x16e9, 0x03f4: 0x16ec, 0x03f5: 0x16f0, + 0x03f6: 0x16f4, 0x03f7: 0x16f8, 0x03f8: 0x16fc, 0x03f9: 0x1700, 0x03fa: 0x1704, 0x03fb: 0x1708, + 0x03fc: 0x170c, 0x03fd: 0x1710, 0x03fe: 0x1715, 0x03ff: 0x171a, // Block 0x10, offset 0x400 - 0x0410: 0x05b1, 0x0411: 0x05b6, - 0x0413: 0x05bb, 0x0417: 0x05c0, - 0x041c: 0x05c5, 0x041d: 0x05ca, - 0x041e: 0x05cf, - 0x0436: 0x05d4, 0x0437: 0x05d9, + 0x0400: 0x171f, 0x0401: 0x1724, 0x0402: 0x1729, 0x0403: 0x172e, 0x0404: 0x1733, 0x0405: 0x1738, + 0x0406: 0x173d, 0x0407: 0x1742, 0x0408: 0x1747, 0x0409: 0x174a, 0x040a: 0x174d, 0x040b: 0x1750, + 0x040c: 0x1753, 0x040d: 0x1756, 0x040e: 0x1759, 0x040f: 0x175c, 0x0410: 0x175f, 0x0411: 0x1762, + 0x0412: 0x1766, 0x0413: 0x176a, 0x0414: 0x176e, 0x0415: 0x1772, 0x0416: 0x1776, 0x0417: 0x177a, + 0x0418: 0x177e, 0x0419: 0x1782, 0x041a: 0x1786, 0x041b: 0x178a, 0x041c: 0x178e, 0x041d: 0x1792, + 0x041e: 0x1796, 0x041f: 0x179a, 0x0420: 0x179e, 0x0421: 0x17a2, 0x0422: 0x17a6, 0x0423: 0x17aa, + 0x0424: 0x17ae, 0x0425: 0x17b2, 0x0426: 0x17b6, 0x0427: 0x17ba, 0x0428: 0x17be, 0x0429: 0x17c2, + 0x042a: 0x17c6, 0x042b: 0x17ca, 0x042c: 0x17ce, 0x042d: 0x17d2, 0x042e: 0x17d6, 0x042f: 0x17da, + 0x0430: 0x17de, 0x0431: 0x17e2, 0x0432: 0x17e6, 0x0433: 0x17ea, 0x0434: 0x17ee, 0x0435: 0x17f2, + 0x0436: 0x0906, 0x0437: 0x090b, 0x0438: 0x14ad, 0x0439: 0x090d, 0x043a: 0x090f, 0x043b: 0x14d9, + 0x043c: 0x0914, 0x043d: 0x0916, 0x043e: 0x0918, 0x043f: 0x091a, // Block 0x11, offset 0x440 - 0x0441: 0x05de, 0x0442: 0x05e3, - 0x0450: 0x05e8, 0x0451: 0x05ed, - 0x0452: 0x05f2, 0x0453: 0x05f7, 0x0456: 0x05fc, 0x0457: 0x0601, - 0x045a: 0x0606, 0x045b: 0x060b, 0x045c: 0x0610, 0x045d: 0x0615, - 0x045e: 0x061a, 0x045f: 0x061f, 0x0462: 0x0624, 0x0463: 0x0629, - 0x0464: 0x062e, 0x0465: 0x0633, 0x0466: 0x0638, 0x0467: 0x063d, - 0x046a: 0x0642, 0x046b: 0x0647, 0x046c: 0x064c, 0x046d: 0x0651, 0x046e: 0x0656, 0x046f: 0x065b, - 0x0470: 0x0660, 0x0471: 0x0665, 0x0472: 0x066a, 0x0473: 0x066f, 0x0474: 0x0674, 0x0475: 0x0679, - 0x0478: 0x067e, 0x0479: 0x0683, + 0x0440: 0x091c, 0x0441: 0x091e, 0x0442: 0x0920, 0x0443: 0x0922, 0x0444: 0x0924, 0x0445: 0x0929, + 0x0446: 0x14c8, 0x0447: 0x092b, 0x0448: 0x17f6, 0x0449: 0x092d, 0x044a: 0x092f, 0x044b: 0x155f, + 0x044c: 0x0931, 0x044d: 0x1570, 0x044e: 0x17f8, 0x044f: 0x14d4, 0x0450: 0x0007, 0x0451: 0x093d, + 0x0452: 0x0984, 0x0453: 0x093f, 0x0454: 0x0941, 0x0455: 0x098c, 0x0456: 0x094c, 0x0457: 0x0494, + 0x0458: 0x097c, 0x0459: 0x0499, 0x045a: 0x094e, 0x045b: 0x04c5, 0x045c: 0x0950, 0x045d: 0x14a0, + 0x045e: 0x001e, 0x045f: 0x0960, 0x0460: 0x17fa, 0x0461: 0x049b, 0x0462: 0x02c8, 0x0463: 0x0962, + 0x0464: 0x0964, 0x0465: 0x096d, 0x0466: 0x04a6, 0x0467: 0x04c7, 0x0468: 0x04a8, 0x0469: 0x09df, + 0x046a: 0x1486, // Block 0x12, offset 0x480 - 0x0487: 0x0688, + 0x048c: 0x1b8a, 0x048e: 0x1b91, 0x0490: 0x1b98, + 0x0492: 0x1b9f, 0x0494: 0x1ba6, 0x0496: 0x1bad, + 0x0498: 0x1bb4, 0x049a: 0x1bbb, 0x049c: 0x1bc2, + 0x049e: 0x1bc9, 0x04a0: 0x1bd0, 0x04a2: 0x1bd7, + 0x04a5: 0x1bde, 0x04a7: 0x1be5, 0x04a9: 0x1bec, + 0x04b0: 0x1bf3, 0x04b1: 0x1bfa, 0x04b3: 0x1c01, 0x04b4: 0x1c08, + 0x04b6: 0x1c0f, 0x04b7: 0x1c16, 0x04b9: 0x1c1d, 0x04ba: 0x1c24, + 0x04bc: 0x1c2b, 0x04bd: 0x1c32, // Block 0x13, offset 0x4c0 - 0x04e2: 0x068d, 0x04e3: 0x0692, - 0x04e4: 0x0697, 0x04e5: 0x069c, 0x04e6: 0x06a1, + 0x04c0: 0x1ed8, 0x04c1: 0x1ede, 0x04c2: 0x1ee4, 0x04c3: 0x1eea, 0x04c4: 0x1ef0, 0x04c5: 0x1ef6, + 0x04c6: 0x1efc, 0x04c7: 0x1f02, 0x04c8: 0x1f08, 0x04c9: 0x1f0e, 0x04ca: 0x1f14, 0x04cb: 0x1f1a, + 0x04cc: 0x1f20, 0x04cd: 0x1f26, 0x04ce: 0x1f2c, 0x04cf: 0x1f35, 0x04d0: 0x1f3e, 0x04d1: 0x1f47, + 0x04d2: 0x1f50, 0x04d3: 0x1f59, 0x04d4: 0x1f62, 0x04d5: 0x1f6b, 0x04d6: 0x1f74, 0x04d7: 0x1f7d, + 0x04d8: 0x1f86, 0x04d9: 0x1f8f, 0x04da: 0x1f98, 0x04db: 0x1fa1, 0x04dc: 0x1faa, 0x04dd: 0x1fb3, + 0x04de: 0x1fc5, 0x04e0: 0x1fd4, 0x04e1: 0x1fda, 0x04e2: 0x1fe0, 0x04e3: 0x1fe6, + 0x04e4: 0x1fec, 0x04e5: 0x1ff2, 0x04e6: 0x1ff8, 0x04e7: 0x1ffe, 0x04e8: 0x2004, 0x04e9: 0x200a, + 0x04ea: 0x2010, 0x04eb: 0x2016, 0x04ec: 0x201c, 0x04ed: 0x2022, 0x04ee: 0x2028, 0x04ef: 0x202e, + 0x04f0: 0x2034, 0x04f1: 0x203a, 0x04f2: 0x2040, 0x04f3: 0x2046, 0x04f4: 0x204c, 0x04f5: 0x2052, + 0x04f6: 0x2058, 0x04f7: 0x205e, 0x04f8: 0x2064, 0x04f9: 0x206a, 0x04fa: 0x2070, 0x04fb: 0x2076, + 0x04fc: 0x207c, 0x04fd: 0x2082, 0x04fe: 0x2088, 0x04ff: 0x208e, // Block 0x14, offset 0x500 - 0x0535: 0x06a6, - 0x0536: 0x06ab, 0x0537: 0x06b0, 0x0538: 0x06b5, + 0x0500: 0x2094, 0x0501: 0x209a, 0x0502: 0x20a0, 0x0503: 0x20a6, 0x0504: 0x20ac, 0x0505: 0x20b0, + 0x0506: 0x192e, 0x0507: 0x20b4, + 0x0510: 0x20b8, 0x0511: 0x20bc, + 0x0512: 0x20bf, 0x0513: 0x20c2, 0x0514: 0x20c5, 0x0515: 0x20c8, 0x0516: 0x20cb, 0x0517: 0x20ce, + 0x0518: 0x20d1, 0x0519: 0x20d4, 0x051a: 0x20d7, 0x051b: 0x20da, 0x051c: 0x20dd, 0x051d: 0x20e0, + 0x051e: 0x20e3, 0x051f: 0x20e6, 0x0520: 0x1d38, 0x0521: 0x1d44, 0x0522: 0x1d50, 0x0523: 0x1d58, + 0x0524: 0x1d78, 0x0525: 0x1d7c, 0x0526: 0x1d88, 0x0527: 0x1d90, 0x0528: 0x1d94, 0x0529: 0x1d9c, + 0x052a: 0x1da0, 0x052b: 0x1da4, 0x052c: 0x1da8, 0x052d: 0x1dac, 0x052e: 0x20e9, 0x052f: 0x20f0, + 0x0530: 0x20f7, 0x0531: 0x20fe, 0x0532: 0x2105, 0x0533: 0x210c, 0x0534: 0x2113, 0x0535: 0x211a, + 0x0536: 0x2121, 0x0537: 0x2128, 0x0538: 0x212f, 0x0539: 0x2136, 0x053a: 0x213d, 0x053b: 0x2144, + 0x053c: 0x214b, 0x053d: 0x215b, 0x053e: 0x2168, // Block 0x15, offset 0x540 - 0x0540: 0x06ba, 0x0542: 0x06bf, - 0x0553: 0x06c4, + 0x0540: 0x1826, 0x0541: 0x183e, 0x0542: 0x1eb0, 0x0543: 0x1eb4, 0x0544: 0x216f, 0x0545: 0x2173, + 0x0546: 0x2177, 0x0547: 0x1852, 0x0548: 0x217b, 0x0549: 0x1882, 0x054a: 0x194a, 0x054b: 0x197a, + 0x054c: 0x1976, 0x054d: 0x194e, 0x054e: 0x1abe, 0x054f: 0x18a2, 0x0550: 0x1942, 0x0551: 0x217f, + 0x0552: 0x2183, 0x0553: 0x2187, 0x0554: 0x218b, 0x0555: 0x218f, 0x0556: 0x2193, 0x0557: 0x2197, + 0x0558: 0x219b, 0x0559: 0x219f, 0x055a: 0x21a3, 0x055b: 0x18ba, 0x055c: 0x21a7, 0x055d: 0x21ab, + 0x055e: 0x21af, 0x055f: 0x21b3, 0x0560: 0x21b7, 0x0561: 0x21bb, 0x0562: 0x21bf, 0x0563: 0x21c3, + 0x0564: 0x1eb8, 0x0565: 0x1ebc, 0x0566: 0x1ec0, 0x0567: 0x21c7, 0x0568: 0x21cb, 0x0569: 0x21cf, + 0x056a: 0x21d3, 0x056b: 0x21d7, 0x056c: 0x21db, 0x056d: 0x21df, 0x056e: 0x21e3, 0x056f: 0x21e7, + 0x0570: 0x21eb, 0x0571: 0x21ef, 0x0572: 0x21f2, 0x0573: 0x21f5, 0x0574: 0x21f8, 0x0575: 0x21fb, + 0x0576: 0x21fe, 0x0577: 0x2201, 0x0578: 0x2204, 0x0579: 0x2207, 0x057a: 0x220a, 0x057b: 0x220d, + 0x057c: 0x2210, 0x057d: 0x2213, 0x057e: 0x2216, 0x057f: 0x2219, // Block 0x16, offset 0x580 - 0x05a9: 0x06c9, - 0x05b1: 0x06d0, 0x05b4: 0x06d7, + 0x0580: 0x2325, 0x0581: 0x2335, 0x0582: 0x2342, 0x0583: 0x2352, 0x0584: 0x235c, 0x0585: 0x236c, + 0x0586: 0x2376, 0x0587: 0x2380, 0x0588: 0x2393, 0x0589: 0x23a0, 0x058a: 0x23aa, 0x058b: 0x23b4, + 0x058c: 0x23be, 0x058d: 0x23cb, 0x058e: 0x23d8, 0x058f: 0x23e5, 0x0590: 0x23f2, 0x0591: 0x23ff, + 0x0592: 0x240c, 0x0593: 0x2419, 0x0594: 0x242c, 0x0595: 0x2433, 0x0596: 0x2446, 0x0597: 0x2459, + 0x0598: 0x2469, 0x0599: 0x2476, 0x059a: 0x2489, 0x059b: 0x249c, 0x059c: 0x24a9, 0x059d: 0x24b3, + 0x059e: 0x24bd, 0x059f: 0x24ca, 0x05a0: 0x24d7, 0x05a1: 0x24e7, 0x05a2: 0x24f7, 0x05a3: 0x2501, + 0x05a4: 0x250b, 0x05a5: 0x2518, 0x05a6: 0x2522, 0x05a7: 0x252c, 0x05a8: 0x2533, 0x05a9: 0x253a, + 0x05aa: 0x2544, 0x05ab: 0x254e, 0x05ac: 0x2561, 0x05ad: 0x256e, 0x05ae: 0x257e, 0x05af: 0x2591, + 0x05b0: 0x259e, 0x05b1: 0x25a8, 0x05b2: 0x25b2, 0x05b3: 0x25c5, 0x05b4: 0x25d2, 0x05b5: 0x25e5, + 0x05b6: 0x25ef, 0x05b7: 0x25ff, 0x05b8: 0x2609, 0x05b9: 0x2616, 0x05ba: 0x2620, 0x05bb: 0x262d, + 0x05bc: 0x263d, 0x05bd: 0x264a, 0x05be: 0x265a, 0x05bf: 0x2667, // Block 0x17, offset 0x5c0 - 0x05d8: 0x06de, 0x05d9: 0x06e5, 0x05da: 0x06ec, 0x05db: 0x06f3, 0x05dc: 0x06fa, 0x05dd: 0x0701, - 0x05de: 0x0708, 0x05df: 0x070f, + 0x05c0: 0x266e, 0x05c1: 0x267e, 0x05c2: 0x2688, 0x05c3: 0x2692, 0x05c4: 0x269f, 0x05c5: 0x26a9, + 0x05c6: 0x26b3, 0x05c7: 0x26bd, 0x05c8: 0x26cd, 0x05c9: 0x26da, 0x05ca: 0x26e1, 0x05cb: 0x26f4, + 0x05cc: 0x26fe, 0x05cd: 0x270e, 0x05ce: 0x271b, 0x05cf: 0x2728, 0x05d0: 0x2732, 0x05d1: 0x273c, + 0x05d2: 0x2749, 0x05d3: 0x2750, 0x05d4: 0x275d, 0x05d5: 0x276d, 0x05d6: 0x2774, 0x05d7: 0x2787, + 0x05d8: 0x2791, 0x05d9: 0x2796, 0x05da: 0x279b, 0x05db: 0x27a0, 0x05dc: 0x27a5, 0x05dd: 0x27aa, + 0x05de: 0x27af, 0x05df: 0x27b4, 0x05e0: 0x27b9, 0x05e1: 0x27be, 0x05e2: 0x27c3, 0x05e3: 0x27c9, + 0x05e4: 0x27cf, 0x05e5: 0x27d5, 0x05e6: 0x27db, 0x05e7: 0x27e1, 0x05e8: 0x27e7, 0x05e9: 0x27ed, + 0x05ea: 0x27f3, 0x05eb: 0x27f9, 0x05ec: 0x27ff, 0x05ed: 0x2805, 0x05ee: 0x280b, 0x05ef: 0x2811, + 0x05f0: 0x2817, 0x05f1: 0x281d, 0x05f2: 0x2821, 0x05f3: 0x2824, 0x05f4: 0x2827, 0x05f5: 0x282b, + 0x05f6: 0x282e, 0x05f7: 0x2831, 0x05f8: 0x2834, 0x05f9: 0x2838, 0x05fa: 0x283c, 0x05fb: 0x283f, + 0x05fc: 0x2846, 0x05fd: 0x284d, 0x05fe: 0x2854, 0x05ff: 0x285b, // Block 0x18, offset 0x600 - 0x060b: 0x0716, - 0x060c: 0x071d, - 0x061c: 0x0724, 0x061d: 0x072b, - 0x061f: 0x0732, + 0x0600: 0x2868, 0x0601: 0x286b, 0x0602: 0x286e, 0x0603: 0x2872, 0x0604: 0x2875, 0x0605: 0x2878, + 0x0606: 0x287b, 0x0607: 0x287e, 0x0608: 0x2881, 0x0609: 0x2885, 0x060a: 0x288a, 0x060b: 0x288d, + 0x060c: 0x2890, 0x060d: 0x2894, 0x060e: 0x2898, 0x060f: 0x289b, 0x0610: 0x289e, 0x0611: 0x28a1, + 0x0612: 0x28a5, 0x0613: 0x28a9, 0x0614: 0x28ad, 0x0615: 0x28b1, 0x0616: 0x28b5, 0x0617: 0x28b8, + 0x0618: 0x28bb, 0x0619: 0x28be, 0x061a: 0x28c1, 0x061b: 0x28c4, 0x061c: 0x28c8, 0x061d: 0x28cb, + 0x061e: 0x28ce, 0x061f: 0x28d1, 0x0620: 0x28d5, 0x0621: 0x28d9, 0x0622: 0x28dc, 0x0623: 0x28e0, + 0x0624: 0x28e4, 0x0625: 0x28e8, 0x0626: 0x28eb, 0x0627: 0x28ef, 0x0628: 0x28f5, 0x0629: 0x28fc, + 0x062a: 0x28ff, 0x062b: 0x2903, 0x062c: 0x2907, 0x062d: 0x290b, 0x062e: 0x290f, 0x062f: 0x2917, + 0x0630: 0x2920, 0x0631: 0x2923, 0x0632: 0x2926, 0x0633: 0x292a, 0x0634: 0x292d, 0x0635: 0x2930, + 0x0636: 0x2933, 0x0637: 0x2937, 0x0638: 0x293a, 0x0639: 0x293d, 0x063a: 0x2940, 0x063b: 0x2943, + 0x063c: 0x2946, 0x063d: 0x294a, 0x063e: 0x294d, 0x063f: 0x2950, // Block 0x19, offset 0x640 - 0x0673: 0x0739, - 0x0676: 0x0740, + 0x0640: 0x2953, 0x0641: 0x2957, 0x0642: 0x295b, 0x0643: 0x2960, 0x0644: 0x2963, 0x0645: 0x2966, + 0x0646: 0x2969, 0x0647: 0x2970, 0x0648: 0x2974, 0x0649: 0x2977, 0x064a: 0x297a, 0x064b: 0x297d, + 0x064c: 0x2980, 0x064d: 0x2983, 0x064e: 0x2986, 0x064f: 0x2989, 0x0650: 0x298c, 0x0651: 0x298f, + 0x0652: 0x2992, 0x0653: 0x2996, 0x0654: 0x2999, 0x0655: 0x299c, 0x0656: 0x29a0, 0x0657: 0x29a4, + 0x0658: 0x29a7, 0x0659: 0x29ac, 0x065a: 0x29b0, 0x065b: 0x29b3, 0x065c: 0x29b6, 0x065d: 0x29b9, + 0x065e: 0x29bc, 0x065f: 0x29c2, 0x0660: 0x29c8, 0x0661: 0x29cd, 0x0662: 0x29d2, 0x0663: 0x29d7, + 0x0664: 0x29dc, 0x0665: 0x29e1, 0x0666: 0x29e6, 0x0667: 0x29eb, 0x0668: 0x29f0, 0x0669: 0x29f5, + 0x066a: 0x29fb, 0x066b: 0x2a01, 0x066c: 0x2a07, 0x066d: 0x2a0d, 0x066e: 0x2a13, 0x066f: 0x2a19, + 0x0670: 0x2a1f, 0x0671: 0x2a25, 0x0672: 0x2a2b, 0x0673: 0x2a31, 0x0674: 0x2a37, 0x0675: 0x2a3d, + 0x0676: 0x2a43, 0x0677: 0x2a49, 0x0678: 0x2a4f, 0x0679: 0x2a55, 0x067a: 0x2a5b, 0x067b: 0x2a61, + 0x067c: 0x2a67, 0x067d: 0x2a6d, 0x067e: 0x2a73, 0x067f: 0x2a79, // Block 0x1a, offset 0x680 - 0x0699: 0x0747, 0x069a: 0x074e, 0x069b: 0x0755, - 0x069e: 0x075c, + 0x0680: 0x2fce, 0x0681: 0x2fd2, 0x0682: 0x2fd6, 0x0683: 0x2fda, 0x0684: 0x2fde, 0x0685: 0x2fe2, + 0x0686: 0x2fe6, 0x0687: 0x2fea, 0x0688: 0x2fee, 0x0689: 0x2eed, 0x068a: 0x2ff2, 0x068b: 0x2ef1, + 0x068c: 0x2ff6, 0x068d: 0x2ffa, 0x068e: 0x2ffe, 0x068f: 0x3002, 0x0690: 0x3006, 0x0691: 0x2e6d, + 0x0692: 0x2b15, 0x0693: 0x300a, 0x0694: 0x300e, 0x0695: 0x195a, 0x0696: 0x2c25, 0x0697: 0x2d71, + 0x0698: 0x3012, 0x0699: 0x3016, 0x069a: 0x2f0d, 0x069b: 0x301a, 0x069c: 0x2f11, 0x069d: 0x301e, + 0x069e: 0x3022, 0x069f: 0x3026, 0x06a0: 0x2e75, 0x06a1: 0x302a, 0x06a2: 0x302e, 0x06a3: 0x3032, + 0x06a4: 0x3036, 0x06a5: 0x303a, 0x06a6: 0x2e79, 0x06a7: 0x303e, 0x06a8: 0x3042, 0x06a9: 0x3046, + 0x06aa: 0x304a, 0x06ab: 0x304e, 0x06ac: 0x3052, 0x06ad: 0x2f41, 0x06ae: 0x3056, 0x06af: 0x305a, + 0x06b0: 0x2cb1, 0x06b1: 0x305e, 0x06b2: 0x2f51, 0x06b3: 0x3062, 0x06b4: 0x3066, 0x06b5: 0x306a, + 0x06b6: 0x306e, 0x06b7: 0x3072, 0x06b8: 0x2f65, 0x06b9: 0x3076, 0x06ba: 0x2e99, 0x06bb: 0x307a, + 0x06bc: 0x2f69, 0x06bd: 0x2bd9, 0x06be: 0x307e, 0x06bf: 0x2f6d, // Block 0x1b, offset 0x6c0 - 0x06c8: 0x0763, 0x06cb: 0x076a, - 0x06cc: 0x0771, - 0x06dc: 0x0778, 0x06dd: 0x077f, + 0x06c0: 0x3082, 0x06c1: 0x2f75, 0x06c2: 0x3086, 0x06c3: 0x308a, 0x06c4: 0x308e, 0x06c5: 0x3092, + 0x06c6: 0x3096, 0x06c7: 0x2f7d, 0x06c8: 0x2e8d, 0x06c9: 0x309a, 0x06ca: 0x2f81, 0x06cb: 0x309e, + 0x06cc: 0x2f85, 0x06cd: 0x30a2, 0x06ce: 0x1b76, 0x06cf: 0x30a6, 0x06d0: 0x30ab, 0x06d1: 0x30b0, + 0x06d2: 0x30b5, 0x06d3: 0x30b9, 0x06d4: 0x30bd, 0x06d5: 0x30c1, 0x06d6: 0x30c6, 0x06d7: 0x30cb, + 0x06d8: 0x30d0, 0x06d9: 0x30d4, // Block 0x1c, offset 0x700 - 0x0714: 0x0786, + 0x0700: 0x30d8, 0x0701: 0x30db, 0x0702: 0x30de, 0x0703: 0x30e1, 0x0704: 0x30e5, 0x0705: 0x30e9, + 0x0706: 0x30e9, + 0x0713: 0x30ec, 0x0714: 0x30f1, 0x0715: 0x30f6, 0x0716: 0x30fb, 0x0717: 0x3100, + 0x071d: 0x3105, + 0x071f: 0x310a, 0x0720: 0x310f, 0x0721: 0x14db, 0x0722: 0x14e4, 0x0723: 0x3112, + 0x0724: 0x3115, 0x0725: 0x3118, 0x0726: 0x311b, 0x0727: 0x311e, 0x0728: 0x3121, 0x0729: 0x1494, + 0x072a: 0x3124, 0x072b: 0x3129, 0x072c: 0x312e, 0x072d: 0x3135, 0x072e: 0x313c, 0x072f: 0x3141, + 0x0730: 0x3146, 0x0731: 0x314b, 0x0732: 0x3150, 0x0733: 0x3155, 0x0734: 0x315a, 0x0735: 0x315f, + 0x0736: 0x3164, 0x0738: 0x3169, 0x0739: 0x316e, 0x073a: 0x3173, 0x073b: 0x3178, + 0x073c: 0x317d, 0x073e: 0x3182, // Block 0x1d, offset 0x740 - 0x074a: 0x078d, 0x074b: 0x0794, - 0x074c: 0x079b, + 0x0740: 0x3187, 0x0741: 0x318c, 0x0743: 0x3191, 0x0744: 0x3196, + 0x0746: 0x319b, 0x0747: 0x31a0, 0x0748: 0x31a5, 0x0749: 0x31aa, 0x074a: 0x31af, 0x074b: 0x31b4, + 0x074c: 0x31b9, 0x074d: 0x31be, 0x074e: 0x31c3, 0x074f: 0x31c8, 0x0750: 0x31cd, 0x0751: 0x31cd, + 0x0752: 0x31d0, 0x0753: 0x31d0, 0x0754: 0x31d0, 0x0755: 0x31d0, 0x0756: 0x31d3, 0x0757: 0x31d3, + 0x0758: 0x31d3, 0x0759: 0x31d3, 0x075a: 0x31d6, 0x075b: 0x31d6, 0x075c: 0x31d6, 0x075d: 0x31d6, + 0x075e: 0x31d9, 0x075f: 0x31d9, 0x0760: 0x31d9, 0x0761: 0x31d9, 0x0762: 0x31dc, 0x0763: 0x31dc, + 0x0764: 0x31dc, 0x0765: 0x31dc, 0x0766: 0x31df, 0x0767: 0x31df, 0x0768: 0x31df, 0x0769: 0x31df, + 0x076a: 0x31e2, 0x076b: 0x31e2, 0x076c: 0x31e2, 0x076d: 0x31e2, 0x076e: 0x31e5, 0x076f: 0x31e5, + 0x0770: 0x31e5, 0x0771: 0x31e5, 0x0772: 0x31e8, 0x0773: 0x31e8, 0x0774: 0x31e8, 0x0775: 0x31e8, + 0x0776: 0x31eb, 0x0777: 0x31eb, 0x0778: 0x31eb, 0x0779: 0x31eb, 0x077a: 0x31ee, 0x077b: 0x31ee, + 0x077c: 0x31ee, 0x077d: 0x31ee, 0x077e: 0x31f1, 0x077f: 0x31f1, // Block 0x1e, offset 0x780 - 0x0788: 0x07a2, + 0x0780: 0x31f1, 0x0781: 0x31f1, 0x0782: 0x31f4, 0x0783: 0x31f4, 0x0784: 0x31f7, 0x0785: 0x31f7, + 0x0786: 0x31fa, 0x0787: 0x31fa, 0x0788: 0x31fd, 0x0789: 0x31fd, 0x078a: 0x3200, 0x078b: 0x3200, + 0x078c: 0x3203, 0x078d: 0x3203, 0x078e: 0x3206, 0x078f: 0x3206, 0x0790: 0x3206, 0x0791: 0x3206, + 0x0792: 0x3209, 0x0793: 0x3209, 0x0794: 0x3209, 0x0795: 0x3209, 0x0796: 0x320c, 0x0797: 0x320c, + 0x0798: 0x320c, 0x0799: 0x320c, 0x079a: 0x320f, 0x079b: 0x320f, 0x079c: 0x320f, 0x079d: 0x320f, + 0x079e: 0x3212, 0x079f: 0x3212, 0x07a0: 0x3215, 0x07a1: 0x3215, 0x07a2: 0x3215, 0x07a3: 0x3215, + 0x07a4: 0x06ba, 0x07a5: 0x06ba, 0x07a6: 0x3218, 0x07a7: 0x3218, 0x07a8: 0x3218, 0x07a9: 0x3218, + 0x07aa: 0x321b, 0x07ab: 0x321b, 0x07ac: 0x321b, 0x07ad: 0x321b, 0x07ae: 0x321e, 0x07af: 0x321e, + 0x07b0: 0x06c4, 0x07b1: 0x06c4, // Block 0x1f, offset 0x7c0 - 0x07c0: 0x07a9, - 0x07c7: 0x07b0, 0x07c8: 0x07b7, 0x07ca: 0x07be, 0x07cb: 0x07c5, + 0x07d3: 0x3221, 0x07d4: 0x3221, 0x07d5: 0x3221, 0x07d6: 0x3221, 0x07d7: 0x3224, + 0x07d8: 0x3224, 0x07d9: 0x3227, 0x07da: 0x3227, 0x07db: 0x322a, 0x07dc: 0x322a, 0x07dd: 0x06b0, + 0x07de: 0x322d, 0x07df: 0x322d, 0x07e0: 0x3230, 0x07e1: 0x3230, 0x07e2: 0x3233, 0x07e3: 0x3233, + 0x07e4: 0x3236, 0x07e5: 0x3236, 0x07e6: 0x3236, 0x07e7: 0x3236, 0x07e8: 0x3239, 0x07e9: 0x3239, + 0x07ea: 0x323c, 0x07eb: 0x323c, 0x07ec: 0x3243, 0x07ed: 0x3243, 0x07ee: 0x324a, 0x07ef: 0x324a, + 0x07f0: 0x3251, 0x07f1: 0x3251, 0x07f2: 0x3258, 0x07f3: 0x3258, 0x07f4: 0x325f, 0x07f5: 0x325f, + 0x07f6: 0x3266, 0x07f7: 0x3266, 0x07f8: 0x3266, 0x07f9: 0x326d, 0x07fa: 0x326d, 0x07fb: 0x326d, + 0x07fc: 0x3274, 0x07fd: 0x3274, 0x07fe: 0x3274, 0x07ff: 0x3274, // Block 0x20, offset 0x800 - 0x080a: 0x07cf, 0x080b: 0x07d6, - 0x080c: 0x07dd, + 0x0800: 0x33ba, 0x0801: 0x33bf, 0x0802: 0x33c4, 0x0803: 0x33c9, 0x0804: 0x33ce, 0x0805: 0x33d3, + 0x0806: 0x33d8, 0x0807: 0x33dd, 0x0808: 0x33e2, 0x0809: 0x33e7, 0x080a: 0x33ec, 0x080b: 0x33f1, + 0x080c: 0x33f6, 0x080d: 0x33fb, 0x080e: 0x3400, 0x080f: 0x3405, 0x0810: 0x340a, 0x0811: 0x340f, + 0x0812: 0x3414, 0x0813: 0x3419, 0x0814: 0x341e, 0x0815: 0x3423, 0x0816: 0x3428, 0x0817: 0x342d, + 0x0818: 0x3432, 0x0819: 0x3437, 0x081a: 0x343c, 0x081b: 0x3441, 0x081c: 0x3446, 0x081d: 0x344b, + 0x081e: 0x3450, 0x081f: 0x3456, 0x0820: 0x345c, 0x0821: 0x3462, 0x0822: 0x3468, 0x0823: 0x346e, + 0x0824: 0x3474, 0x0825: 0x347b, 0x0826: 0x3285, 0x0827: 0x3482, 0x0828: 0x326d, 0x0829: 0x328c, + 0x082a: 0x3489, 0x082b: 0x348e, 0x082c: 0x32a2, 0x082d: 0x3493, 0x082e: 0x32a7, 0x082f: 0x32ac, + 0x0830: 0x3498, 0x0831: 0x349d, 0x0832: 0x32c0, 0x0833: 0x34a2, 0x0834: 0x32c5, 0x0835: 0x32ca, + 0x0836: 0x34a7, 0x0837: 0x34ac, 0x0838: 0x32d4, 0x0839: 0x34b1, 0x083a: 0x32d9, 0x083b: 0x32de, + 0x083c: 0x336f, 0x083d: 0x3374, 0x083e: 0x3383, 0x083f: 0x3388, // Block 0x21, offset 0x840 - 0x085a: 0x07e4, 0x085c: 0x07eb, 0x085d: 0x07f2, - 0x085e: 0x07fc, + 0x0840: 0x338d, 0x0841: 0x33a1, 0x0842: 0x33a6, 0x0843: 0x33ab, 0x0844: 0x33b0, 0x0845: 0x33c4, + 0x0846: 0x33c9, 0x0847: 0x33ce, 0x0848: 0x34b6, 0x0849: 0x33e2, 0x084a: 0x34bb, 0x084b: 0x34c0, + 0x084c: 0x3400, 0x084d: 0x34c5, 0x084e: 0x3405, 0x084f: 0x340a, 0x0850: 0x344b, 0x0851: 0x34ca, + 0x0852: 0x34cf, 0x0853: 0x3432, 0x0854: 0x34d4, 0x0855: 0x3437, 0x0856: 0x343c, 0x0857: 0x3277, + 0x0858: 0x327e, 0x0859: 0x34d9, 0x085a: 0x3285, 0x085b: 0x34e0, 0x085c: 0x3293, 0x085d: 0x3298, + 0x085e: 0x329d, 0x085f: 0x32a2, 0x0860: 0x34e7, 0x0861: 0x32b1, 0x0862: 0x32b6, 0x0863: 0x32bb, + 0x0864: 0x32c0, 0x0865: 0x34ec, 0x0866: 0x32d4, 0x0867: 0x32e3, 0x0868: 0x32e8, 0x0869: 0x32ed, + 0x086a: 0x32f2, 0x086b: 0x32f7, 0x086c: 0x3301, 0x086d: 0x3306, 0x086e: 0x330b, 0x086f: 0x3310, + 0x0870: 0x3315, 0x0871: 0x331a, 0x0872: 0x34f1, 0x0873: 0x331f, 0x0874: 0x3324, 0x0875: 0x3329, + 0x0876: 0x332e, 0x0877: 0x3333, 0x0878: 0x3338, 0x0879: 0x3342, 0x087a: 0x3347, 0x087b: 0x334c, + 0x087c: 0x3351, 0x087d: 0x3356, 0x087e: 0x335b, 0x087f: 0x3360, // Block 0x22, offset 0x880 - 0x08b3: 0x0803, + 0x0880: 0x3365, 0x0881: 0x336a, 0x0882: 0x3379, 0x0883: 0x337e, 0x0884: 0x3392, 0x0885: 0x3397, + 0x0886: 0x339c, 0x0887: 0x33a1, 0x0888: 0x33a6, 0x0889: 0x33b5, 0x088a: 0x33ba, 0x088b: 0x33bf, + 0x088c: 0x33c4, 0x088d: 0x34f6, 0x088e: 0x33d3, 0x088f: 0x33d8, 0x0890: 0x33dd, 0x0891: 0x33e2, + 0x0892: 0x33f1, 0x0893: 0x33f6, 0x0894: 0x33fb, 0x0895: 0x3400, 0x0896: 0x34fb, 0x0897: 0x340f, + 0x0898: 0x3414, 0x0899: 0x3500, 0x089a: 0x3423, 0x089b: 0x3428, 0x089c: 0x342d, 0x089d: 0x3432, + 0x089e: 0x3505, 0x089f: 0x3285, 0x08a0: 0x34e0, 0x08a1: 0x32a2, 0x08a2: 0x34e7, 0x08a3: 0x32c0, + 0x08a4: 0x34ec, 0x08a5: 0x32d4, 0x08a6: 0x350a, 0x08a7: 0x3315, 0x08a8: 0x350f, 0x08a9: 0x3514, + 0x08aa: 0x3519, 0x08ab: 0x33a1, 0x08ac: 0x33a6, 0x08ad: 0x33c4, 0x08ae: 0x3400, 0x08af: 0x34fb, + 0x08b0: 0x3432, 0x08b1: 0x3505, 0x08b2: 0x351e, 0x08b3: 0x3525, 0x08b4: 0x352c, 0x08b5: 0x3533, + 0x08b6: 0x3538, 0x08b7: 0x353d, 0x08b8: 0x3542, 0x08b9: 0x3547, 0x08ba: 0x354c, 0x08bb: 0x3551, + 0x08bc: 0x3556, 0x08bd: 0x355b, 0x08be: 0x3560, 0x08bf: 0x3565, // Block 0x23, offset 0x8c0 - 0x08f3: 0x080a, + 0x08c0: 0x16c7, 0x08c1: 0x391e, 0x08c2: 0x3922, 0x08c3: 0x3926, 0x08c4: 0x392a, + 0x08c7: 0x392e, 0x08c8: 0x3930, 0x08c9: 0x146c, 0x08ca: 0x146c, 0x08cb: 0x146c, + 0x08cc: 0x146c, 0x08cd: 0x3900, 0x08ce: 0x3900, 0x08cf: 0x3900, 0x08d0: 0x38e0, 0x08d1: 0x38e2, + 0x08d2: 0x143e, 0x08d4: 0x04e1, 0x08d5: 0x38ea, 0x08d6: 0x38ee, 0x08d7: 0x38ec, + 0x08d8: 0x38f8, 0x08d9: 0x149c, 0x08da: 0x149e, 0x08db: 0x3902, 0x08dc: 0x3904, 0x08dd: 0x3906, + 0x08de: 0x390a, 0x08df: 0x3932, 0x08e0: 0x3934, 0x08e1: 0x3936, 0x08e2: 0x1494, 0x08e3: 0x3938, + 0x08e4: 0x393a, 0x08e5: 0x393c, 0x08e6: 0x149a, 0x08e8: 0x393e, 0x08e9: 0x3940, + 0x08ea: 0x3942, 0x08eb: 0x3944, + 0x08f0: 0x3946, 0x08f1: 0x394a, 0x08f2: 0x394f, 0x08f4: 0x3953, + 0x08f6: 0x3957, 0x08f7: 0x395b, 0x08f8: 0x3960, 0x08f9: 0x3964, 0x08fa: 0x3969, 0x08fb: 0x396d, + 0x08fc: 0x3972, 0x08fd: 0x3976, 0x08fe: 0x397b, 0x08ff: 0x397f, // Block 0x24, offset 0x900 - 0x091c: 0x0811, 0x091d: 0x0818, + 0x0900: 0x3984, 0x0901: 0x068d, 0x0902: 0x068d, 0x0903: 0x0692, 0x0904: 0x0692, 0x0905: 0x0697, + 0x0906: 0x0697, 0x0907: 0x069c, 0x0908: 0x069c, 0x0909: 0x06a1, 0x090a: 0x06a1, 0x090b: 0x06a1, + 0x090c: 0x06a1, 0x090d: 0x3987, 0x090e: 0x3987, 0x090f: 0x398a, 0x0910: 0x398a, 0x0911: 0x398a, + 0x0912: 0x398a, 0x0913: 0x398d, 0x0914: 0x398d, 0x0915: 0x3990, 0x0916: 0x3990, 0x0917: 0x3990, + 0x0918: 0x3990, 0x0919: 0x3993, 0x091a: 0x3993, 0x091b: 0x3993, 0x091c: 0x3993, 0x091d: 0x3996, + 0x091e: 0x3996, 0x091f: 0x3996, 0x0920: 0x3996, 0x0921: 0x3999, 0x0922: 0x3999, 0x0923: 0x3999, + 0x0924: 0x3999, 0x0925: 0x399c, 0x0926: 0x399c, 0x0927: 0x399c, 0x0928: 0x399c, 0x0929: 0x399f, + 0x092a: 0x399f, 0x092b: 0x39a2, 0x092c: 0x39a2, 0x092d: 0x39a5, 0x092e: 0x39a5, 0x092f: 0x39a8, + 0x0930: 0x39a8, 0x0931: 0x39ab, 0x0932: 0x39ab, 0x0933: 0x39ab, 0x0934: 0x39ab, 0x0935: 0x39ae, + 0x0936: 0x39ae, 0x0937: 0x39ae, 0x0938: 0x39ae, 0x0939: 0x39b1, 0x093a: 0x39b1, 0x093b: 0x39b1, + 0x093c: 0x39b1, 0x093d: 0x39b4, 0x093e: 0x39b4, 0x093f: 0x39b4, // Block 0x25, offset 0x940 - 0x094c: 0x081f, + 0x0940: 0x39b4, 0x0941: 0x39b7, 0x0942: 0x39b7, 0x0943: 0x39b7, 0x0944: 0x39b7, 0x0945: 0x39ba, + 0x0946: 0x39ba, 0x0947: 0x39ba, 0x0948: 0x39ba, 0x0949: 0x39bd, 0x094a: 0x39bd, 0x094b: 0x39bd, + 0x094c: 0x39bd, 0x094d: 0x39c0, 0x094e: 0x39c0, 0x094f: 0x39c0, 0x0950: 0x39c0, 0x0951: 0x39c3, + 0x0952: 0x39c3, 0x0953: 0x39c3, 0x0954: 0x39c3, 0x0955: 0x39c6, 0x0956: 0x39c6, 0x0957: 0x39c6, + 0x0958: 0x39c6, 0x0959: 0x39c9, 0x095a: 0x39c9, 0x095b: 0x39c9, 0x095c: 0x39c9, 0x095d: 0x39cc, + 0x095e: 0x39cc, 0x095f: 0x39cc, 0x0960: 0x39cc, 0x0961: 0x39cf, 0x0962: 0x39cf, 0x0963: 0x39cf, + 0x0964: 0x39cf, 0x0965: 0x39d2, 0x0966: 0x39d2, 0x0967: 0x39d2, 0x0968: 0x39d2, 0x0969: 0x39d5, + 0x096a: 0x39d5, 0x096b: 0x39d5, 0x096c: 0x39d5, 0x096d: 0x39d8, 0x096e: 0x39d8, 0x096f: 0x3239, + 0x0970: 0x3239, 0x0971: 0x39db, 0x0972: 0x39db, 0x0973: 0x39db, 0x0974: 0x39db, 0x0975: 0x39de, + 0x0976: 0x39de, 0x0977: 0x39e5, 0x0978: 0x39e5, 0x0979: 0x39ec, 0x097a: 0x39ec, 0x097b: 0x39f3, + 0x097c: 0x39f3, // Block 0x26, offset 0x980 - 0x0983: 0x0823, - 0x098d: 0x082a, - 0x0992: 0x0831, 0x0997: 0x0838, - 0x099c: 0x083f, - 0x09a9: 0x0846, - 0x09b3: 0x084d, 0x09b5: 0x0854, - 0x09b6: 0x085b, 0x09b7: 0x0862, 0x09b8: 0x086c, 0x09b9: 0x0873, + 0x0981: 0x38ec, 0x0982: 0x39f8, 0x0983: 0x3932, 0x0984: 0x3940, 0x0985: 0x3942, + 0x0986: 0x3934, 0x0987: 0x39fa, 0x0988: 0x149c, 0x0989: 0x149e, 0x098a: 0x3936, 0x098b: 0x1494, + 0x098c: 0x38e0, 0x098d: 0x3938, 0x098e: 0x143e, 0x098f: 0x39fc, 0x0990: 0x1486, 0x0991: 0x001c, + 0x0992: 0x000d, 0x0993: 0x000f, 0x0994: 0x1488, 0x0995: 0x148a, 0x0996: 0x148c, 0x0997: 0x148e, + 0x0998: 0x1490, 0x0999: 0x1492, 0x099a: 0x38ea, 0x099b: 0x04e1, 0x099c: 0x393a, 0x099d: 0x149a, + 0x099e: 0x393c, 0x099f: 0x38ee, 0x09a0: 0x3944, 0x09a1: 0x0906, 0x09a2: 0x090b, 0x09a3: 0x14ad, + 0x09a4: 0x090d, 0x09a5: 0x090f, 0x09a6: 0x14d9, 0x09a7: 0x0914, 0x09a8: 0x0916, 0x09a9: 0x0918, + 0x09aa: 0x091a, 0x09ab: 0x091c, 0x09ac: 0x091e, 0x09ad: 0x0920, 0x09ae: 0x0922, 0x09af: 0x0924, + 0x09b0: 0x0929, 0x09b1: 0x14c8, 0x09b2: 0x092b, 0x09b3: 0x17f6, 0x09b4: 0x092d, 0x09b5: 0x092f, + 0x09b6: 0x155f, 0x09b7: 0x0931, 0x09b8: 0x1570, 0x09b9: 0x17f8, 0x09ba: 0x14d4, 0x09bb: 0x392e, + 0x09bc: 0x393e, 0x09bd: 0x3930, 0x09be: 0x39fe, 0x09bf: 0x3900, // Block 0x27, offset 0x9c0 - 0x09c1: 0x087d, - 0x09d3: 0x0884, - 0x09dd: 0x088b, - 0x09e2: 0x0892, - 0x09e7: 0x0899, - 0x09ec: 0x08a0, - 0x09f9: 0x08a7, + 0x09c0: 0x13f7, 0x09c1: 0x0007, 0x09c2: 0x093d, 0x09c3: 0x0984, 0x09c4: 0x093f, 0x09c5: 0x0941, + 0x09c6: 0x098c, 0x09c7: 0x094c, 0x09c8: 0x0494, 0x09c9: 0x097c, 0x09ca: 0x0499, 0x09cb: 0x094e, + 0x09cc: 0x04c5, 0x09cd: 0x0950, 0x09ce: 0x14a0, 0x09cf: 0x001e, 0x09d0: 0x0960, 0x09d1: 0x17fa, + 0x09d2: 0x049b, 0x09d3: 0x02c8, 0x09d4: 0x0962, 0x09d5: 0x0964, 0x09d6: 0x096d, 0x09d7: 0x04a6, + 0x09d8: 0x04c7, 0x09d9: 0x04a8, 0x09da: 0x09df, 0x09db: 0x3902, 0x09dc: 0x3a00, 0x09dd: 0x3904, + 0x09de: 0x3a02, 0x09df: 0x3a04, 0x09e0: 0x3a08, 0x09e1: 0x38e6, 0x09e2: 0x391e, 0x09e3: 0x3922, + 0x09e4: 0x38e2, 0x09e5: 0x3a0c, 0x09e6: 0x2321, 0x09e7: 0x3a10, 0x09e8: 0x3a14, 0x09e9: 0x3a18, + 0x09ea: 0x3a1c, 0x09eb: 0x3a20, 0x09ec: 0x3a24, 0x09ed: 0x3a28, 0x09ee: 0x3a2c, 0x09ef: 0x3a30, + 0x09f0: 0x3a34, 0x09f1: 0x2269, 0x09f2: 0x226d, 0x09f3: 0x2271, 0x09f4: 0x2275, 0x09f5: 0x2279, + 0x09f6: 0x227d, 0x09f7: 0x2281, 0x09f8: 0x2285, 0x09f9: 0x2289, 0x09fa: 0x228d, 0x09fb: 0x2291, + 0x09fc: 0x2295, 0x09fd: 0x2299, 0x09fe: 0x229d, 0x09ff: 0x22a1, // Block 0x28, offset 0xa00 - 0x0a26: 0x08ae, + 0x0a00: 0x0906, 0x0a01: 0x090b, 0x0a02: 0x14ad, 0x0a03: 0x090d, 0x0a04: 0x090f, 0x0a05: 0x14d9, + 0x0a06: 0x0914, 0x0a07: 0x0916, 0x0a08: 0x0918, 0x0a09: 0x091a, 0x0a0a: 0x091c, 0x0a0b: 0x091e, + 0x0a0c: 0x0920, 0x0a0d: 0x0922, 0x0a0e: 0x0924, 0x0a0f: 0x0929, 0x0a10: 0x14c8, 0x0a11: 0x092b, + 0x0a12: 0x17f6, 0x0a13: 0x092d, 0x0a14: 0x092f, 0x0a15: 0x155f, 0x0a16: 0x0931, 0x0a17: 0x1570, + 0x0a18: 0x17f8, 0x0a19: 0x14d4, 0x0a1a: 0x0007, 0x0a1b: 0x093d, 0x0a1c: 0x0984, 0x0a1d: 0x093f, + 0x0a1e: 0x0941, 0x0a1f: 0x098c, 0x0a20: 0x094c, 0x0a21: 0x0494, 0x0a22: 0x097c, 0x0a23: 0x0499, + 0x0a24: 0x094e, 0x0a25: 0x04c5, 0x0a26: 0x0950, 0x0a27: 0x14a0, 0x0a28: 0x001e, 0x0a29: 0x0960, + 0x0a2a: 0x17fa, 0x0a2b: 0x049b, 0x0a2c: 0x02c8, 0x0a2d: 0x0962, 0x0a2e: 0x0964, 0x0a2f: 0x096d, + 0x0a30: 0x04a6, 0x0a31: 0x04c7, 0x0a32: 0x04a8, 0x0a33: 0x09df, 0x0a34: 0x0906, 0x0a35: 0x090b, + 0x0a36: 0x14ad, 0x0a37: 0x090d, 0x0a38: 0x090f, 0x0a39: 0x14d9, 0x0a3a: 0x0914, 0x0a3b: 0x0916, + 0x0a3c: 0x0918, 0x0a3d: 0x091a, 0x0a3e: 0x091c, 0x0a3f: 0x091e, // Block 0x29, offset 0xa40 - 0x0a7c: 0x08b5, + 0x0a40: 0x0920, 0x0a41: 0x0922, 0x0a42: 0x0924, 0x0a43: 0x0929, 0x0a44: 0x14c8, 0x0a45: 0x092b, + 0x0a46: 0x17f6, 0x0a47: 0x092d, 0x0a48: 0x092f, 0x0a49: 0x155f, 0x0a4a: 0x0931, 0x0a4b: 0x1570, + 0x0a4c: 0x17f8, 0x0a4d: 0x14d4, 0x0a4e: 0x0007, 0x0a4f: 0x093d, 0x0a50: 0x0984, 0x0a51: 0x093f, + 0x0a52: 0x0941, 0x0a53: 0x098c, 0x0a54: 0x094c, 0x0a56: 0x097c, 0x0a57: 0x0499, + 0x0a58: 0x094e, 0x0a59: 0x04c5, 0x0a5a: 0x0950, 0x0a5b: 0x14a0, 0x0a5c: 0x001e, 0x0a5d: 0x0960, + 0x0a5e: 0x17fa, 0x0a5f: 0x049b, 0x0a60: 0x02c8, 0x0a61: 0x0962, 0x0a62: 0x0964, 0x0a63: 0x096d, + 0x0a64: 0x04a6, 0x0a65: 0x04c7, 0x0a66: 0x04a8, 0x0a67: 0x09df, 0x0a68: 0x0906, 0x0a69: 0x090b, + 0x0a6a: 0x14ad, 0x0a6b: 0x090d, 0x0a6c: 0x090f, 0x0a6d: 0x14d9, 0x0a6e: 0x0914, 0x0a6f: 0x0916, + 0x0a70: 0x0918, 0x0a71: 0x091a, 0x0a72: 0x091c, 0x0a73: 0x091e, 0x0a74: 0x0920, 0x0a75: 0x0922, + 0x0a76: 0x0924, 0x0a77: 0x0929, 0x0a78: 0x14c8, 0x0a79: 0x092b, 0x0a7a: 0x17f6, 0x0a7b: 0x092d, + 0x0a7c: 0x092f, 0x0a7d: 0x155f, 0x0a7e: 0x0931, 0x0a7f: 0x1570, // Block 0x2a, offset 0xa80 - 0x0a86: 0x08b9, 0x0a88: 0x08c0, 0x0a8a: 0x08c7, - 0x0a8c: 0x08ce, 0x0a8e: 0x08d5, - 0x0a92: 0x08dc, - 0x0abb: 0x08e3, - 0x0abd: 0x08ea, + 0x0a80: 0x17f8, 0x0a81: 0x14d4, 0x0a82: 0x0007, 0x0a83: 0x093d, 0x0a84: 0x0984, 0x0a85: 0x093f, + 0x0a86: 0x0941, 0x0a87: 0x098c, 0x0a88: 0x094c, 0x0a89: 0x0494, 0x0a8a: 0x097c, 0x0a8b: 0x0499, + 0x0a8c: 0x094e, 0x0a8d: 0x04c5, 0x0a8e: 0x0950, 0x0a8f: 0x14a0, 0x0a90: 0x001e, 0x0a91: 0x0960, + 0x0a92: 0x17fa, 0x0a93: 0x049b, 0x0a94: 0x02c8, 0x0a95: 0x0962, 0x0a96: 0x0964, 0x0a97: 0x096d, + 0x0a98: 0x04a6, 0x0a99: 0x04c7, 0x0a9a: 0x04a8, 0x0a9b: 0x09df, 0x0a9c: 0x0906, + 0x0a9e: 0x14ad, 0x0a9f: 0x090d, 0x0aa2: 0x0914, + 0x0aa5: 0x091a, 0x0aa6: 0x091c, 0x0aa9: 0x0922, + 0x0aaa: 0x0924, 0x0aab: 0x0929, 0x0aac: 0x14c8, 0x0aae: 0x17f6, 0x0aaf: 0x092d, + 0x0ab0: 0x092f, 0x0ab1: 0x155f, 0x0ab2: 0x0931, 0x0ab3: 0x1570, 0x0ab4: 0x17f8, 0x0ab5: 0x14d4, + 0x0ab6: 0x0007, 0x0ab7: 0x093d, 0x0ab8: 0x0984, 0x0ab9: 0x093f, 0x0abb: 0x098c, + 0x0abd: 0x0494, 0x0abe: 0x097c, 0x0abf: 0x0499, // Block 0x2b, offset 0xac0 - 0x0ac0: 0x08f1, 0x0ac1: 0x08f8, 0x0ac3: 0x08ff, + 0x0ac0: 0x094e, 0x0ac1: 0x04c5, 0x0ac2: 0x0950, 0x0ac3: 0x14a0, 0x0ac5: 0x0960, + 0x0ac6: 0x17fa, 0x0ac7: 0x049b, 0x0ac8: 0x02c8, 0x0ac9: 0x0962, 0x0aca: 0x0964, 0x0acb: 0x096d, + 0x0acc: 0x04a6, 0x0acd: 0x04c7, 0x0ace: 0x04a8, 0x0acf: 0x09df, 0x0ad0: 0x0906, 0x0ad1: 0x090b, + 0x0ad2: 0x14ad, 0x0ad3: 0x090d, 0x0ad4: 0x090f, 0x0ad5: 0x14d9, 0x0ad6: 0x0914, 0x0ad7: 0x0916, + 0x0ad8: 0x0918, 0x0ad9: 0x091a, 0x0ada: 0x091c, 0x0adb: 0x091e, 0x0adc: 0x0920, 0x0add: 0x0922, + 0x0ade: 0x0924, 0x0adf: 0x0929, 0x0ae0: 0x14c8, 0x0ae1: 0x092b, 0x0ae2: 0x17f6, 0x0ae3: 0x092d, + 0x0ae4: 0x092f, 0x0ae5: 0x155f, 0x0ae6: 0x0931, 0x0ae7: 0x1570, 0x0ae8: 0x17f8, 0x0ae9: 0x14d4, + 0x0aea: 0x0007, 0x0aeb: 0x093d, 0x0aec: 0x0984, 0x0aed: 0x093f, 0x0aee: 0x0941, 0x0aef: 0x098c, + 0x0af0: 0x094c, 0x0af1: 0x0494, 0x0af2: 0x097c, 0x0af3: 0x0499, 0x0af4: 0x094e, 0x0af5: 0x04c5, + 0x0af6: 0x0950, 0x0af7: 0x14a0, 0x0af8: 0x001e, 0x0af9: 0x0960, 0x0afa: 0x17fa, 0x0afb: 0x049b, + 0x0afc: 0x02c8, 0x0afd: 0x0962, 0x0afe: 0x0964, 0x0aff: 0x096d, // Block 0x2c, offset 0xb00 - 0x0b2c: 0x0906, 0x0b2d: 0x0908, 0x0b2e: 0x090b, - 0x0b30: 0x090d, 0x0b31: 0x090f, 0x0b32: 0x0911, 0x0b33: 0x0914, 0x0b34: 0x0916, 0x0b35: 0x0918, - 0x0b36: 0x091a, 0x0b37: 0x091c, 0x0b38: 0x091e, 0x0b39: 0x0920, 0x0b3a: 0x0922, - 0x0b3c: 0x0924, 0x0b3d: 0x0926, 0x0b3e: 0x0929, 0x0b3f: 0x092b, + 0x0b00: 0x04a6, 0x0b01: 0x04c7, 0x0b02: 0x04a8, 0x0b03: 0x09df, 0x0b04: 0x0906, 0x0b05: 0x090b, + 0x0b07: 0x090d, 0x0b08: 0x090f, 0x0b09: 0x14d9, 0x0b0a: 0x0914, + 0x0b0d: 0x091a, 0x0b0e: 0x091c, 0x0b0f: 0x091e, 0x0b10: 0x0920, 0x0b11: 0x0922, + 0x0b12: 0x0924, 0x0b13: 0x0929, 0x0b14: 0x14c8, 0x0b16: 0x17f6, 0x0b17: 0x092d, + 0x0b18: 0x092f, 0x0b19: 0x155f, 0x0b1a: 0x0931, 0x0b1b: 0x1570, 0x0b1c: 0x17f8, + 0x0b1e: 0x0007, 0x0b1f: 0x093d, 0x0b20: 0x0984, 0x0b21: 0x093f, 0x0b22: 0x0941, 0x0b23: 0x098c, + 0x0b24: 0x094c, 0x0b25: 0x0494, 0x0b26: 0x097c, 0x0b27: 0x0499, 0x0b28: 0x094e, 0x0b29: 0x04c5, + 0x0b2a: 0x0950, 0x0b2b: 0x14a0, 0x0b2c: 0x001e, 0x0b2d: 0x0960, 0x0b2e: 0x17fa, 0x0b2f: 0x049b, + 0x0b30: 0x02c8, 0x0b31: 0x0962, 0x0b32: 0x0964, 0x0b33: 0x096d, 0x0b34: 0x04a6, 0x0b35: 0x04c7, + 0x0b36: 0x04a8, 0x0b37: 0x09df, 0x0b38: 0x0906, 0x0b39: 0x090b, 0x0b3b: 0x090d, + 0x0b3c: 0x090f, 0x0b3d: 0x14d9, 0x0b3e: 0x0914, // Block 0x2d, offset 0xb40 - 0x0b40: 0x092d, 0x0b41: 0x092f, 0x0b42: 0x0931, 0x0b43: 0x0007, 0x0b44: 0x0933, 0x0b45: 0x0936, - 0x0b46: 0x0939, 0x0b47: 0x093d, 0x0b48: 0x093f, 0x0b49: 0x0941, 0x0b4a: 0x0943, 0x0b4b: 0x0946, - 0x0b4c: 0x0949, 0x0b4d: 0x094c, 0x0b4f: 0x094e, 0x0b50: 0x0950, 0x0b51: 0x0952, - 0x0b52: 0x001e, 0x0b53: 0x0955, 0x0b54: 0x0958, 0x0b55: 0x095c, 0x0b56: 0x0960, 0x0b57: 0x0962, - 0x0b58: 0x0964, 0x0b59: 0x0966, 0x0b5a: 0x096a, 0x0b5b: 0x096d, 0x0b5c: 0x096f, 0x0b5d: 0x0559, - 0x0b5e: 0x0973, 0x0b5f: 0x0976, 0x0b60: 0x056c, 0x0b61: 0x0979, 0x0b62: 0x097c, 0x0b63: 0x049b, - 0x0b64: 0x0964, 0x0b65: 0x096d, 0x0b66: 0x0559, 0x0b67: 0x0973, 0x0b68: 0x0575, 0x0b69: 0x056c, - 0x0b6a: 0x0979, - 0x0b78: 0x097e, + 0x0b40: 0x0918, 0x0b41: 0x091a, 0x0b42: 0x091c, 0x0b43: 0x091e, 0x0b44: 0x0920, + 0x0b46: 0x0924, 0x0b4a: 0x17f6, 0x0b4b: 0x092d, + 0x0b4c: 0x092f, 0x0b4d: 0x155f, 0x0b4e: 0x0931, 0x0b4f: 0x1570, 0x0b50: 0x17f8, + 0x0b52: 0x0007, 0x0b53: 0x093d, 0x0b54: 0x0984, 0x0b55: 0x093f, 0x0b56: 0x0941, 0x0b57: 0x098c, + 0x0b58: 0x094c, 0x0b59: 0x0494, 0x0b5a: 0x097c, 0x0b5b: 0x0499, 0x0b5c: 0x094e, 0x0b5d: 0x04c5, + 0x0b5e: 0x0950, 0x0b5f: 0x14a0, 0x0b60: 0x001e, 0x0b61: 0x0960, 0x0b62: 0x17fa, 0x0b63: 0x049b, + 0x0b64: 0x02c8, 0x0b65: 0x0962, 0x0b66: 0x0964, 0x0b67: 0x096d, 0x0b68: 0x04a6, 0x0b69: 0x04c7, + 0x0b6a: 0x04a8, 0x0b6b: 0x09df, 0x0b6c: 0x0906, 0x0b6d: 0x090b, 0x0b6e: 0x14ad, 0x0b6f: 0x090d, + 0x0b70: 0x090f, 0x0b71: 0x14d9, 0x0b72: 0x0914, 0x0b73: 0x0916, 0x0b74: 0x0918, 0x0b75: 0x091a, + 0x0b76: 0x091c, 0x0b77: 0x091e, 0x0b78: 0x0920, 0x0b79: 0x0922, 0x0b7a: 0x0924, 0x0b7b: 0x0929, + 0x0b7c: 0x14c8, 0x0b7d: 0x092b, 0x0b7e: 0x17f6, 0x0b7f: 0x092d, // Block 0x2e, offset 0xb80 - 0x0b9b: 0x0981, 0x0b9c: 0x0984, 0x0b9d: 0x0986, - 0x0b9e: 0x0989, 0x0b9f: 0x0949, 0x0ba0: 0x098c, 0x0ba1: 0x098e, 0x0ba2: 0x0991, 0x0ba3: 0x0994, - 0x0ba4: 0x0997, 0x0ba5: 0x099a, 0x0ba6: 0x099d, 0x0ba7: 0x09a0, 0x0ba8: 0x09a4, 0x0ba9: 0x09a7, - 0x0baa: 0x09aa, 0x0bab: 0x09ae, 0x0bac: 0x09b1, 0x0bad: 0x09b4, 0x0bae: 0x09b7, 0x0baf: 0x09ba, - 0x0bb0: 0x09bd, 0x0bb1: 0x09c0, 0x0bb2: 0x09c3, 0x0bb3: 0x09c6, 0x0bb4: 0x09c9, 0x0bb5: 0x09cc, - 0x0bb6: 0x09cf, 0x0bb7: 0x09d2, 0x0bb8: 0x09d5, 0x0bb9: 0x09d9, 0x0bba: 0x09dc, 0x0bbb: 0x09df, - 0x0bbc: 0x09e1, 0x0bbd: 0x09e4, 0x0bbe: 0x09e7, 0x0bbf: 0x055c, + 0x0b80: 0x092f, 0x0b81: 0x155f, 0x0b82: 0x0931, 0x0b83: 0x1570, 0x0b84: 0x17f8, 0x0b85: 0x14d4, + 0x0b86: 0x0007, 0x0b87: 0x093d, 0x0b88: 0x0984, 0x0b89: 0x093f, 0x0b8a: 0x0941, 0x0b8b: 0x098c, + 0x0b8c: 0x094c, 0x0b8d: 0x0494, 0x0b8e: 0x097c, 0x0b8f: 0x0499, 0x0b90: 0x094e, 0x0b91: 0x04c5, + 0x0b92: 0x0950, 0x0b93: 0x14a0, 0x0b94: 0x001e, 0x0b95: 0x0960, 0x0b96: 0x17fa, 0x0b97: 0x049b, + 0x0b98: 0x02c8, 0x0b99: 0x0962, 0x0b9a: 0x0964, 0x0b9b: 0x096d, 0x0b9c: 0x04a6, 0x0b9d: 0x04c7, + 0x0b9e: 0x04a8, 0x0b9f: 0x09df, 0x0ba0: 0x0906, 0x0ba1: 0x090b, 0x0ba2: 0x14ad, 0x0ba3: 0x090d, + 0x0ba4: 0x090f, 0x0ba5: 0x14d9, 0x0ba6: 0x0914, 0x0ba7: 0x0916, 0x0ba8: 0x0918, 0x0ba9: 0x091a, + 0x0baa: 0x091c, 0x0bab: 0x091e, 0x0bac: 0x0920, 0x0bad: 0x0922, 0x0bae: 0x0924, 0x0baf: 0x0929, + 0x0bb0: 0x14c8, 0x0bb1: 0x092b, 0x0bb2: 0x17f6, 0x0bb3: 0x092d, 0x0bb4: 0x092f, 0x0bb5: 0x155f, + 0x0bb6: 0x0931, 0x0bb7: 0x1570, 0x0bb8: 0x17f8, 0x0bb9: 0x14d4, 0x0bba: 0x0007, 0x0bbb: 0x093d, + 0x0bbc: 0x0984, 0x0bbd: 0x093f, 0x0bbe: 0x0941, 0x0bbf: 0x098c, // Block 0x2f, offset 0xbc0 - 0x0bc0: 0x09ea, 0x0bc1: 0x09ee, 0x0bc2: 0x09f2, 0x0bc3: 0x09f6, 0x0bc4: 0x09fa, 0x0bc5: 0x09fe, - 0x0bc6: 0x0a02, 0x0bc7: 0x0a06, 0x0bc8: 0x0a0a, 0x0bc9: 0x0a10, 0x0bca: 0x0a16, 0x0bcb: 0x0a1a, - 0x0bcc: 0x0a1e, 0x0bcd: 0x0a22, 0x0bce: 0x0a26, 0x0bcf: 0x0a2a, 0x0bd0: 0x0a2e, 0x0bd1: 0x0a32, - 0x0bd2: 0x0a36, 0x0bd3: 0x0a3a, 0x0bd4: 0x0a3e, 0x0bd5: 0x0a44, 0x0bd6: 0x0a4a, 0x0bd7: 0x0a50, - 0x0bd8: 0x0a56, 0x0bd9: 0x0a5a, 0x0bda: 0x0a5e, 0x0bdb: 0x0a62, 0x0bdc: 0x0a66, 0x0bdd: 0x0a6c, - 0x0bde: 0x0a72, 0x0bdf: 0x0a76, 0x0be0: 0x0a7a, 0x0be1: 0x0a7e, 0x0be2: 0x0a82, 0x0be3: 0x0a86, - 0x0be4: 0x0a8a, 0x0be5: 0x0a8e, 0x0be6: 0x0a92, 0x0be7: 0x0a96, 0x0be8: 0x0a9a, 0x0be9: 0x0a9e, - 0x0bea: 0x0aa2, 0x0beb: 0x0aa6, 0x0bec: 0x0aaa, 0x0bed: 0x0aae, 0x0bee: 0x0ab2, 0x0bef: 0x0ab8, - 0x0bf0: 0x0abe, 0x0bf1: 0x0ac2, 0x0bf2: 0x0ac6, 0x0bf3: 0x0aca, 0x0bf4: 0x0ace, 0x0bf5: 0x0ad2, - 0x0bf6: 0x0ad6, 0x0bf7: 0x0ada, 0x0bf8: 0x0ade, 0x0bf9: 0x0ae4, 0x0bfa: 0x0aea, 0x0bfb: 0x0aee, - 0x0bfc: 0x0af2, 0x0bfd: 0x0af6, 0x0bfe: 0x0afa, 0x0bff: 0x0afe, + 0x0bc0: 0x094c, 0x0bc1: 0x0494, 0x0bc2: 0x097c, 0x0bc3: 0x0499, 0x0bc4: 0x094e, 0x0bc5: 0x04c5, + 0x0bc6: 0x0950, 0x0bc7: 0x14a0, 0x0bc8: 0x001e, 0x0bc9: 0x0960, 0x0bca: 0x17fa, 0x0bcb: 0x049b, + 0x0bcc: 0x02c8, 0x0bcd: 0x0962, 0x0bce: 0x0964, 0x0bcf: 0x096d, 0x0bd0: 0x04a6, 0x0bd1: 0x04c7, + 0x0bd2: 0x04a8, 0x0bd3: 0x09df, 0x0bd4: 0x0906, 0x0bd5: 0x090b, 0x0bd6: 0x14ad, 0x0bd7: 0x090d, + 0x0bd8: 0x090f, 0x0bd9: 0x14d9, 0x0bda: 0x0914, 0x0bdb: 0x0916, 0x0bdc: 0x0918, 0x0bdd: 0x091a, + 0x0bde: 0x091c, 0x0bdf: 0x091e, 0x0be0: 0x0920, 0x0be1: 0x0922, 0x0be2: 0x0924, 0x0be3: 0x0929, + 0x0be4: 0x14c8, 0x0be5: 0x092b, 0x0be6: 0x17f6, 0x0be7: 0x092d, 0x0be8: 0x092f, 0x0be9: 0x155f, + 0x0bea: 0x0931, 0x0beb: 0x1570, 0x0bec: 0x17f8, 0x0bed: 0x14d4, 0x0bee: 0x0007, 0x0bef: 0x093d, + 0x0bf0: 0x0984, 0x0bf1: 0x093f, 0x0bf2: 0x0941, 0x0bf3: 0x098c, 0x0bf4: 0x094c, 0x0bf5: 0x0494, + 0x0bf6: 0x097c, 0x0bf7: 0x0499, 0x0bf8: 0x094e, 0x0bf9: 0x04c5, 0x0bfa: 0x0950, 0x0bfb: 0x14a0, + 0x0bfc: 0x001e, 0x0bfd: 0x0960, 0x0bfe: 0x17fa, 0x0bff: 0x049b, // Block 0x30, offset 0xc00 - 0x0c00: 0x0b02, 0x0c01: 0x0b06, 0x0c02: 0x0b0a, 0x0c03: 0x0b0e, 0x0c04: 0x0b12, 0x0c05: 0x0b16, - 0x0c06: 0x0b1a, 0x0c07: 0x0b1e, 0x0c08: 0x0b22, 0x0c09: 0x0b26, 0x0c0a: 0x0b2a, 0x0c0b: 0x0b2e, - 0x0c0c: 0x0b32, 0x0c0d: 0x0b38, 0x0c0e: 0x0b3e, 0x0c0f: 0x0b44, 0x0c10: 0x0b4a, 0x0c11: 0x0b50, - 0x0c12: 0x0b56, 0x0c13: 0x0b5c, 0x0c14: 0x0b62, 0x0c15: 0x0b66, 0x0c16: 0x0b6a, 0x0c17: 0x0b6e, - 0x0c18: 0x0b72, 0x0c19: 0x0b76, 0x0c1a: 0x0b7a, 0x0c1b: 0x0b7e, 0x0c1c: 0x0b82, 0x0c1d: 0x0b88, - 0x0c1e: 0x0b8e, 0x0c1f: 0x0b92, 0x0c20: 0x0b96, 0x0c21: 0x0b9a, 0x0c22: 0x0b9e, 0x0c23: 0x0ba2, - 0x0c24: 0x0ba6, 0x0c25: 0x0bac, 0x0c26: 0x0bb2, 0x0c27: 0x0bb8, 0x0c28: 0x0bbe, 0x0c29: 0x0bc4, - 0x0c2a: 0x0bca, 0x0c2b: 0x0bce, 0x0c2c: 0x0bd2, 0x0c2d: 0x0bd6, 0x0c2e: 0x0bda, 0x0c2f: 0x0bde, - 0x0c30: 0x0be2, 0x0c31: 0x0be6, 0x0c32: 0x0bea, 0x0c33: 0x0bee, 0x0c34: 0x0bf2, 0x0c35: 0x0bf6, - 0x0c36: 0x0bfa, 0x0c37: 0x0bfe, 0x0c38: 0x0c02, 0x0c39: 0x0c08, 0x0c3a: 0x0c0e, 0x0c3b: 0x0c14, - 0x0c3c: 0x0c1a, 0x0c3d: 0x0c1e, 0x0c3e: 0x0c22, 0x0c3f: 0x0c26, + 0x0c00: 0x02c8, 0x0c01: 0x0962, 0x0c02: 0x0964, 0x0c03: 0x096d, 0x0c04: 0x04a6, 0x0c05: 0x04c7, + 0x0c06: 0x04a8, 0x0c07: 0x09df, 0x0c08: 0x0906, 0x0c09: 0x090b, 0x0c0a: 0x14ad, 0x0c0b: 0x090d, + 0x0c0c: 0x090f, 0x0c0d: 0x14d9, 0x0c0e: 0x0914, 0x0c0f: 0x0916, 0x0c10: 0x0918, 0x0c11: 0x091a, + 0x0c12: 0x091c, 0x0c13: 0x091e, 0x0c14: 0x0920, 0x0c15: 0x0922, 0x0c16: 0x0924, 0x0c17: 0x0929, + 0x0c18: 0x14c8, 0x0c19: 0x092b, 0x0c1a: 0x17f6, 0x0c1b: 0x092d, 0x0c1c: 0x092f, 0x0c1d: 0x155f, + 0x0c1e: 0x0931, 0x0c1f: 0x1570, 0x0c20: 0x17f8, 0x0c21: 0x14d4, 0x0c22: 0x0007, 0x0c23: 0x093d, + 0x0c24: 0x0984, 0x0c25: 0x093f, 0x0c26: 0x0941, 0x0c27: 0x098c, 0x0c28: 0x094c, 0x0c29: 0x0494, + 0x0c2a: 0x097c, 0x0c2b: 0x0499, 0x0c2c: 0x094e, 0x0c2d: 0x04c5, 0x0c2e: 0x0950, 0x0c2f: 0x14a0, + 0x0c30: 0x001e, 0x0c31: 0x0960, 0x0c32: 0x17fa, 0x0c33: 0x049b, 0x0c34: 0x02c8, 0x0c35: 0x0962, + 0x0c36: 0x0964, 0x0c37: 0x096d, 0x0c38: 0x04a6, 0x0c39: 0x04c7, 0x0c3a: 0x04a8, 0x0c3b: 0x09df, + 0x0c3c: 0x0906, 0x0c3d: 0x090b, 0x0c3e: 0x14ad, 0x0c3f: 0x090d, // Block 0x31, offset 0xc40 - 0x0c40: 0x0c2a, 0x0c41: 0x0c2e, 0x0c42: 0x0c32, 0x0c43: 0x0c36, 0x0c44: 0x0c3a, 0x0c45: 0x0c3e, - 0x0c46: 0x0c42, 0x0c47: 0x0c46, 0x0c48: 0x0c4a, 0x0c49: 0x0c4e, 0x0c4a: 0x0c52, 0x0c4b: 0x0c56, - 0x0c4c: 0x0c5a, 0x0c4d: 0x0c5e, 0x0c4e: 0x0c62, 0x0c4f: 0x0c66, 0x0c50: 0x0c6a, 0x0c51: 0x0c6e, - 0x0c52: 0x0c72, 0x0c53: 0x0c76, 0x0c54: 0x0c7a, 0x0c55: 0x0c7e, 0x0c56: 0x0c82, 0x0c57: 0x0c86, - 0x0c58: 0x0c8a, 0x0c59: 0x0c8e, 0x0c5a: 0x0c92, 0x0c5b: 0x0b9a, - 0x0c60: 0x0c9b, 0x0c61: 0x0c9f, 0x0c62: 0x0ca3, 0x0c63: 0x0ca7, - 0x0c64: 0x0cab, 0x0c65: 0x0cb1, 0x0c66: 0x0cb7, 0x0c67: 0x0cbd, 0x0c68: 0x0cc3, 0x0c69: 0x0cc9, - 0x0c6a: 0x0ccf, 0x0c6b: 0x0cd5, 0x0c6c: 0x0cdb, 0x0c6d: 0x0ce1, 0x0c6e: 0x0ce7, 0x0c6f: 0x0ced, - 0x0c70: 0x0cf3, 0x0c71: 0x0cf9, 0x0c72: 0x0cff, 0x0c73: 0x0d05, 0x0c74: 0x0d0b, 0x0c75: 0x0d11, - 0x0c76: 0x0d17, 0x0c77: 0x0d1d, 0x0c78: 0x0d23, 0x0c79: 0x0d27, 0x0c7a: 0x0d2b, 0x0c7b: 0x0d2f, - 0x0c7c: 0x0d33, 0x0c7d: 0x0d37, 0x0c7e: 0x0d3b, 0x0c7f: 0x0d41, + 0x0c40: 0x090f, 0x0c41: 0x14d9, 0x0c42: 0x0914, 0x0c43: 0x0916, 0x0c44: 0x0918, 0x0c45: 0x091a, + 0x0c46: 0x091c, 0x0c47: 0x091e, 0x0c48: 0x0920, 0x0c49: 0x0922, 0x0c4a: 0x0924, 0x0c4b: 0x0929, + 0x0c4c: 0x14c8, 0x0c4d: 0x092b, 0x0c4e: 0x17f6, 0x0c4f: 0x092d, 0x0c50: 0x092f, 0x0c51: 0x155f, + 0x0c52: 0x0931, 0x0c53: 0x1570, 0x0c54: 0x17f8, 0x0c55: 0x14d4, 0x0c56: 0x0007, 0x0c57: 0x093d, + 0x0c58: 0x0984, 0x0c59: 0x093f, 0x0c5a: 0x0941, 0x0c5b: 0x098c, 0x0c5c: 0x094c, 0x0c5d: 0x0494, + 0x0c5e: 0x097c, 0x0c5f: 0x0499, 0x0c60: 0x094e, 0x0c61: 0x04c5, 0x0c62: 0x0950, 0x0c63: 0x14a0, + 0x0c64: 0x001e, 0x0c65: 0x0960, 0x0c66: 0x17fa, 0x0c67: 0x049b, 0x0c68: 0x02c8, 0x0c69: 0x0962, + 0x0c6a: 0x0964, 0x0c6b: 0x096d, 0x0c6c: 0x04a6, 0x0c6d: 0x04c7, 0x0c6e: 0x04a8, 0x0c6f: 0x09df, + 0x0c70: 0x0906, 0x0c71: 0x090b, 0x0c72: 0x14ad, 0x0c73: 0x090d, 0x0c74: 0x090f, 0x0c75: 0x14d9, + 0x0c76: 0x0914, 0x0c77: 0x0916, 0x0c78: 0x0918, 0x0c79: 0x091a, 0x0c7a: 0x091c, 0x0c7b: 0x091e, + 0x0c7c: 0x0920, 0x0c7d: 0x0922, 0x0c7e: 0x0924, 0x0c7f: 0x0929, // Block 0x32, offset 0xc80 - 0x0c80: 0x0d47, 0x0c81: 0x0d4d, 0x0c82: 0x0d53, 0x0c83: 0x0d59, 0x0c84: 0x0d5f, 0x0c85: 0x0d65, - 0x0c86: 0x0d6b, 0x0c87: 0x0d71, 0x0c88: 0x0d77, 0x0c89: 0x0d7b, 0x0c8a: 0x0d7f, 0x0c8b: 0x0d83, - 0x0c8c: 0x0d87, 0x0c8d: 0x0d8b, 0x0c8e: 0x0d8f, 0x0c8f: 0x0d93, 0x0c90: 0x0d97, 0x0c91: 0x0d9d, - 0x0c92: 0x0da3, 0x0c93: 0x0da9, 0x0c94: 0x0daf, 0x0c95: 0x0db5, 0x0c96: 0x0dbb, 0x0c97: 0x0dc1, - 0x0c98: 0x0dc7, 0x0c99: 0x0dcd, 0x0c9a: 0x0dd3, 0x0c9b: 0x0dd9, 0x0c9c: 0x0ddf, 0x0c9d: 0x0de5, - 0x0c9e: 0x0deb, 0x0c9f: 0x0df1, 0x0ca0: 0x0df7, 0x0ca1: 0x0dfd, 0x0ca2: 0x0e03, 0x0ca3: 0x0e09, - 0x0ca4: 0x0e0f, 0x0ca5: 0x0e13, 0x0ca6: 0x0e17, 0x0ca7: 0x0e1b, 0x0ca8: 0x0e1f, 0x0ca9: 0x0e25, - 0x0caa: 0x0e2b, 0x0cab: 0x0e31, 0x0cac: 0x0e37, 0x0cad: 0x0e3d, 0x0cae: 0x0e43, 0x0caf: 0x0e49, - 0x0cb0: 0x0e4f, 0x0cb1: 0x0e55, 0x0cb2: 0x0e5b, 0x0cb3: 0x0e5f, 0x0cb4: 0x0e63, 0x0cb5: 0x0e67, - 0x0cb6: 0x0e6b, 0x0cb7: 0x0e6f, 0x0cb8: 0x0e73, 0x0cb9: 0x0e77, + 0x0c80: 0x14c8, 0x0c81: 0x092b, 0x0c82: 0x17f6, 0x0c83: 0x092d, 0x0c84: 0x092f, 0x0c85: 0x155f, + 0x0c86: 0x0931, 0x0c87: 0x1570, 0x0c88: 0x17f8, 0x0c89: 0x14d4, 0x0c8a: 0x0007, 0x0c8b: 0x093d, + 0x0c8c: 0x0984, 0x0c8d: 0x093f, 0x0c8e: 0x0941, 0x0c8f: 0x098c, 0x0c90: 0x094c, 0x0c91: 0x0494, + 0x0c92: 0x097c, 0x0c93: 0x0499, 0x0c94: 0x094e, 0x0c95: 0x04c5, 0x0c96: 0x0950, 0x0c97: 0x14a0, + 0x0c98: 0x001e, 0x0c99: 0x0960, 0x0c9a: 0x17fa, 0x0c9b: 0x049b, 0x0c9c: 0x02c8, 0x0c9d: 0x0962, + 0x0c9e: 0x0964, 0x0c9f: 0x096d, 0x0ca0: 0x04a6, 0x0ca1: 0x04c7, 0x0ca2: 0x04a8, 0x0ca3: 0x09df, + 0x0ca4: 0x3b27, 0x0ca5: 0x3b2a, 0x0ca8: 0x3b2d, 0x0ca9: 0x3b30, + 0x0caa: 0x14eb, 0x0cab: 0x3b33, 0x0cac: 0x3b36, 0x0cad: 0x3b39, 0x0cae: 0x3b3c, 0x0caf: 0x057b, + 0x0cb0: 0x3b3f, 0x0cb1: 0x3b42, 0x0cb2: 0x3b45, 0x0cb3: 0x3b48, 0x0cb4: 0x3b4b, 0x0cb5: 0x3b4e, + 0x0cb6: 0x3b51, 0x0cb7: 0x14ee, 0x0cb8: 0x3b54, 0x0cb9: 0x057b, 0x0cba: 0x0581, 0x0cbb: 0x3b57, + 0x0cbc: 0x055f, 0x0cbd: 0x3b5a, 0x0cbe: 0x3b5d, 0x0cbf: 0x3b60, // Block 0x33, offset 0xcc0 - 0x0cc0: 0x0e7b, 0x0cc1: 0x0e80, 0x0cc2: 0x0e85, 0x0cc3: 0x0e8c, 0x0cc4: 0x0e93, 0x0cc5: 0x0e9a, - 0x0cc6: 0x0ea1, 0x0cc7: 0x0ea8, 0x0cc8: 0x0eaf, 0x0cc9: 0x0eb4, 0x0cca: 0x0eb9, 0x0ccb: 0x0ec0, - 0x0ccc: 0x0ec7, 0x0ccd: 0x0ece, 0x0cce: 0x0ed5, 0x0ccf: 0x0edc, 0x0cd0: 0x0ee3, 0x0cd1: 0x0ee8, - 0x0cd2: 0x0eed, 0x0cd3: 0x0ef4, 0x0cd4: 0x0efb, 0x0cd5: 0x0f02, - 0x0cd8: 0x0f09, 0x0cd9: 0x0f0e, 0x0cda: 0x0f13, 0x0cdb: 0x0f1a, 0x0cdc: 0x0f21, 0x0cdd: 0x0f28, - 0x0ce0: 0x0f2f, 0x0ce1: 0x0f34, 0x0ce2: 0x0f39, 0x0ce3: 0x0f40, - 0x0ce4: 0x0f47, 0x0ce5: 0x0f4e, 0x0ce6: 0x0f55, 0x0ce7: 0x0f5c, 0x0ce8: 0x0f63, 0x0ce9: 0x0f68, - 0x0cea: 0x0f6d, 0x0ceb: 0x0f74, 0x0cec: 0x0f7b, 0x0ced: 0x0f82, 0x0cee: 0x0f89, 0x0cef: 0x0f90, - 0x0cf0: 0x0f97, 0x0cf1: 0x0f9c, 0x0cf2: 0x0fa1, 0x0cf3: 0x0fa8, 0x0cf4: 0x0faf, 0x0cf5: 0x0fb6, - 0x0cf6: 0x0fbd, 0x0cf7: 0x0fc4, 0x0cf8: 0x0fcb, 0x0cf9: 0x0fd0, 0x0cfa: 0x0fd5, 0x0cfb: 0x0fdc, - 0x0cfc: 0x0fe3, 0x0cfd: 0x0fea, 0x0cfe: 0x0ff1, 0x0cff: 0x0ff8, + 0x0cc0: 0x14d6, 0x0cc1: 0x3b63, 0x0cc2: 0x3b67, 0x0cc3: 0x0559, 0x0cc4: 0x0973, 0x0cc5: 0x0976, + 0x0cc6: 0x057e, 0x0cc7: 0x3b6a, 0x0cc8: 0x3b6d, 0x0cc9: 0x055c, 0x0cca: 0x12fd, 0x0ccb: 0x0572, + 0x0ccc: 0x3b70, 0x0ccd: 0x0015, 0x0cce: 0x3b73, 0x0ccf: 0x3b76, 0x0cd0: 0x3b79, 0x0cd1: 0x056f, + 0x0cd2: 0x0575, 0x0cd3: 0x0578, 0x0cd4: 0x3b7c, 0x0cd5: 0x3b7f, 0x0cd6: 0x3b82, 0x0cd7: 0x056c, + 0x0cd8: 0x0979, 0x0cd9: 0x3b85, 0x0cda: 0x3b88, 0x0cdb: 0x3b8b, 0x0cdc: 0x057e, 0x0cdd: 0x055c, + 0x0cde: 0x0572, 0x0cdf: 0x056c, 0x0ce0: 0x0575, 0x0ce1: 0x056f, 0x0ce2: 0x3b2d, 0x0ce3: 0x3b30, + 0x0ce4: 0x14eb, 0x0ce5: 0x3b33, 0x0ce6: 0x3b36, 0x0ce7: 0x3b39, 0x0ce8: 0x3b3c, 0x0ce9: 0x057b, + 0x0cea: 0x3b3f, 0x0ceb: 0x3b42, 0x0cec: 0x3b45, 0x0ced: 0x3b48, 0x0cee: 0x3b4b, 0x0cef: 0x3b4e, + 0x0cf0: 0x3b51, 0x0cf1: 0x14ee, 0x0cf2: 0x3b54, 0x0cf3: 0x057b, 0x0cf4: 0x0581, 0x0cf5: 0x3b57, + 0x0cf6: 0x055f, 0x0cf7: 0x3b5a, 0x0cf8: 0x3b5d, 0x0cf9: 0x3b60, 0x0cfa: 0x14d6, 0x0cfb: 0x3b63, + 0x0cfc: 0x3b67, 0x0cfd: 0x0559, 0x0cfe: 0x0973, 0x0cff: 0x0976, // Block 0x34, offset 0xd00 - 0x0d00: 0x0fff, 0x0d01: 0x1004, 0x0d02: 0x1009, 0x0d03: 0x1010, 0x0d04: 0x1017, 0x0d05: 0x101e, - 0x0d08: 0x1025, 0x0d09: 0x102a, 0x0d0a: 0x102f, 0x0d0b: 0x1036, - 0x0d0c: 0x103d, 0x0d0d: 0x1044, 0x0d10: 0x104b, 0x0d11: 0x1050, - 0x0d12: 0x1055, 0x0d13: 0x105c, 0x0d14: 0x1063, 0x0d15: 0x106a, 0x0d16: 0x1071, 0x0d17: 0x1078, - 0x0d19: 0x107f, 0x0d1b: 0x1084, 0x0d1d: 0x108b, - 0x0d1f: 0x1092, 0x0d20: 0x1099, 0x0d21: 0x109e, 0x0d22: 0x10a3, 0x0d23: 0x10aa, - 0x0d24: 0x10b1, 0x0d25: 0x10b8, 0x0d26: 0x10bf, 0x0d27: 0x10c6, 0x0d28: 0x10cd, 0x0d29: 0x10d2, - 0x0d2a: 0x10d7, 0x0d2b: 0x10de, 0x0d2c: 0x10e5, 0x0d2d: 0x10ec, 0x0d2e: 0x10f3, 0x0d2f: 0x10fa, - 0x0d30: 0x1101, 0x0d31: 0x0525, 0x0d32: 0x1106, 0x0d33: 0x052a, 0x0d34: 0x110b, 0x0d35: 0x052f, - 0x0d36: 0x1110, 0x0d37: 0x0534, 0x0d38: 0x1115, 0x0d39: 0x054a, 0x0d3a: 0x111a, 0x0d3b: 0x054f, - 0x0d3c: 0x111f, 0x0d3d: 0x0554, + 0x0d00: 0x057e, 0x0d01: 0x3b6a, 0x0d02: 0x3b6d, 0x0d03: 0x055c, 0x0d04: 0x12fd, 0x0d05: 0x0572, + 0x0d06: 0x3b70, 0x0d07: 0x0015, 0x0d08: 0x3b73, 0x0d09: 0x3b76, 0x0d0a: 0x3b79, 0x0d0b: 0x056f, + 0x0d0c: 0x0575, 0x0d0d: 0x0578, 0x0d0e: 0x3b7c, 0x0d0f: 0x3b7f, 0x0d10: 0x3b82, 0x0d11: 0x056c, + 0x0d12: 0x0979, 0x0d13: 0x3b85, 0x0d14: 0x3b88, 0x0d15: 0x3b8b, 0x0d16: 0x057e, 0x0d17: 0x055c, + 0x0d18: 0x0572, 0x0d19: 0x056c, 0x0d1a: 0x0575, 0x0d1b: 0x056f, 0x0d1c: 0x3b2d, 0x0d1d: 0x3b30, + 0x0d1e: 0x14eb, 0x0d1f: 0x3b33, 0x0d20: 0x3b36, 0x0d21: 0x3b39, 0x0d22: 0x3b3c, 0x0d23: 0x057b, + 0x0d24: 0x3b3f, 0x0d25: 0x3b42, 0x0d26: 0x3b45, 0x0d27: 0x3b48, 0x0d28: 0x3b4b, 0x0d29: 0x3b4e, + 0x0d2a: 0x3b51, 0x0d2b: 0x14ee, 0x0d2c: 0x3b54, 0x0d2d: 0x057b, 0x0d2e: 0x0581, 0x0d2f: 0x3b57, + 0x0d30: 0x055f, 0x0d31: 0x3b5a, 0x0d32: 0x3b5d, 0x0d33: 0x3b60, 0x0d34: 0x14d6, 0x0d35: 0x3b63, + 0x0d36: 0x3b67, 0x0d37: 0x0559, 0x0d38: 0x0973, 0x0d39: 0x0976, 0x0d3a: 0x057e, 0x0d3b: 0x3b6a, + 0x0d3c: 0x3b6d, 0x0d3d: 0x055c, 0x0d3e: 0x12fd, 0x0d3f: 0x0572, // Block 0x35, offset 0xd40 - 0x0d40: 0x1124, 0x0d41: 0x112b, 0x0d42: 0x1132, 0x0d43: 0x113b, 0x0d44: 0x1144, 0x0d45: 0x114d, - 0x0d46: 0x1156, 0x0d47: 0x115f, 0x0d48: 0x1168, 0x0d49: 0x116f, 0x0d4a: 0x1176, 0x0d4b: 0x117f, - 0x0d4c: 0x1188, 0x0d4d: 0x1191, 0x0d4e: 0x119a, 0x0d4f: 0x11a3, 0x0d50: 0x11ac, 0x0d51: 0x11b3, - 0x0d52: 0x11ba, 0x0d53: 0x11c3, 0x0d54: 0x11cc, 0x0d55: 0x11d5, 0x0d56: 0x11de, 0x0d57: 0x11e7, - 0x0d58: 0x11f0, 0x0d59: 0x11f7, 0x0d5a: 0x11fe, 0x0d5b: 0x1207, 0x0d5c: 0x1210, 0x0d5d: 0x1219, - 0x0d5e: 0x1222, 0x0d5f: 0x122b, 0x0d60: 0x1234, 0x0d61: 0x123b, 0x0d62: 0x1242, 0x0d63: 0x124b, - 0x0d64: 0x1254, 0x0d65: 0x125d, 0x0d66: 0x1266, 0x0d67: 0x126f, 0x0d68: 0x1278, 0x0d69: 0x127f, - 0x0d6a: 0x1286, 0x0d6b: 0x128f, 0x0d6c: 0x1298, 0x0d6d: 0x12a1, 0x0d6e: 0x12aa, 0x0d6f: 0x12b3, - 0x0d70: 0x12bc, 0x0d71: 0x12c1, 0x0d72: 0x12c6, 0x0d73: 0x12cd, 0x0d74: 0x12d2, - 0x0d76: 0x12d9, 0x0d77: 0x12de, 0x0d78: 0x12e5, 0x0d79: 0x12ea, 0x0d7a: 0x12ef, 0x0d7b: 0x04ee, - 0x0d7c: 0x12f4, 0x0d7d: 0x12f9, 0x0d7e: 0x12fd, 0x0d7f: 0x12f9, + 0x0d40: 0x3b70, 0x0d41: 0x0015, 0x0d42: 0x3b73, 0x0d43: 0x3b76, 0x0d44: 0x3b79, 0x0d45: 0x056f, + 0x0d46: 0x0575, 0x0d47: 0x0578, 0x0d48: 0x3b7c, 0x0d49: 0x3b7f, 0x0d4a: 0x3b82, 0x0d4b: 0x056c, + 0x0d4c: 0x0979, 0x0d4d: 0x3b85, 0x0d4e: 0x3b88, 0x0d4f: 0x3b8b, 0x0d50: 0x057e, 0x0d51: 0x055c, + 0x0d52: 0x0572, 0x0d53: 0x056c, 0x0d54: 0x0575, 0x0d55: 0x056f, 0x0d56: 0x3b2d, 0x0d57: 0x3b30, + 0x0d58: 0x14eb, 0x0d59: 0x3b33, 0x0d5a: 0x3b36, 0x0d5b: 0x3b39, 0x0d5c: 0x3b3c, 0x0d5d: 0x057b, + 0x0d5e: 0x3b3f, 0x0d5f: 0x3b42, 0x0d60: 0x3b45, 0x0d61: 0x3b48, 0x0d62: 0x3b4b, 0x0d63: 0x3b4e, + 0x0d64: 0x3b51, 0x0d65: 0x14ee, 0x0d66: 0x3b54, 0x0d67: 0x057b, 0x0d68: 0x0581, 0x0d69: 0x3b57, + 0x0d6a: 0x055f, 0x0d6b: 0x3b5a, 0x0d6c: 0x3b5d, 0x0d6d: 0x3b60, 0x0d6e: 0x14d6, 0x0d6f: 0x3b63, + 0x0d70: 0x3b67, 0x0d71: 0x0559, 0x0d72: 0x0973, 0x0d73: 0x0976, 0x0d74: 0x057e, 0x0d75: 0x3b6a, + 0x0d76: 0x3b6d, 0x0d77: 0x055c, 0x0d78: 0x12fd, 0x0d79: 0x0572, 0x0d7a: 0x3b70, 0x0d7b: 0x0015, + 0x0d7c: 0x3b73, 0x0d7d: 0x3b76, 0x0d7e: 0x3b79, 0x0d7f: 0x056f, // Block 0x36, offset 0xd80 - 0x0d80: 0x1300, 0x0d81: 0x1309, 0x0d82: 0x130f, 0x0d83: 0x1316, 0x0d84: 0x131b, - 0x0d86: 0x1322, 0x0d87: 0x1327, 0x0d88: 0x132e, 0x0d89: 0x04f6, 0x0d8a: 0x1333, 0x0d8b: 0x04fb, - 0x0d8c: 0x1338, 0x0d8d: 0x1343, 0x0d8e: 0x134f, 0x0d8f: 0x135b, 0x0d90: 0x1361, 0x0d91: 0x1366, - 0x0d92: 0x136b, 0x0d93: 0x0514, 0x0d96: 0x1372, 0x0d97: 0x1377, - 0x0d98: 0x137e, 0x0d99: 0x1383, 0x0d9a: 0x1388, 0x0d9b: 0x0500, 0x0d9d: 0x1393, - 0x0d9e: 0x139f, 0x0d9f: 0x13ab, 0x0da0: 0x13b1, 0x0da1: 0x13b6, 0x0da2: 0x13bb, 0x0da3: 0x0539, - 0x0da4: 0x13c2, 0x0da5: 0x13c7, 0x0da6: 0x13cc, 0x0da7: 0x13d1, 0x0da8: 0x13d8, 0x0da9: 0x13dd, - 0x0daa: 0x13e2, 0x0dab: 0x050a, 0x0dac: 0x13e7, 0x0dad: 0x13f1, 0x0dae: 0x04e8, 0x0daf: 0x13f7, - 0x0db2: 0x13f9, 0x0db3: 0x1400, 0x0db4: 0x1405, - 0x0db6: 0x140c, 0x0db7: 0x1411, 0x0db8: 0x1418, 0x0db9: 0x0505, 0x0dba: 0x141d, 0x0dbb: 0x050f, - 0x0dbc: 0x1422, 0x0dbd: 0x0011, 0x0dbe: 0x142a, + 0x0d80: 0x0575, 0x0d81: 0x0578, 0x0d82: 0x3b7c, 0x0d83: 0x3b7f, 0x0d84: 0x3b82, 0x0d85: 0x056c, + 0x0d86: 0x0979, 0x0d87: 0x3b85, 0x0d88: 0x3b88, 0x0d89: 0x3b8b, 0x0d8a: 0x057e, 0x0d8b: 0x055c, + 0x0d8c: 0x0572, 0x0d8d: 0x056c, 0x0d8e: 0x0575, 0x0d8f: 0x056f, 0x0d90: 0x3b2d, 0x0d91: 0x3b30, + 0x0d92: 0x14eb, 0x0d93: 0x3b33, 0x0d94: 0x3b36, 0x0d95: 0x3b39, 0x0d96: 0x3b3c, 0x0d97: 0x057b, + 0x0d98: 0x3b3f, 0x0d99: 0x3b42, 0x0d9a: 0x3b45, 0x0d9b: 0x3b48, 0x0d9c: 0x3b4b, 0x0d9d: 0x3b4e, + 0x0d9e: 0x3b51, 0x0d9f: 0x14ee, 0x0da0: 0x3b54, 0x0da1: 0x057b, 0x0da2: 0x0581, 0x0da3: 0x3b57, + 0x0da4: 0x055f, 0x0da5: 0x3b5a, 0x0da6: 0x3b5d, 0x0da7: 0x3b60, 0x0da8: 0x14d6, 0x0da9: 0x3b63, + 0x0daa: 0x3b67, 0x0dab: 0x0559, 0x0dac: 0x0973, 0x0dad: 0x0976, 0x0dae: 0x057e, 0x0daf: 0x3b6a, + 0x0db0: 0x3b6d, 0x0db1: 0x055c, 0x0db2: 0x12fd, 0x0db3: 0x0572, 0x0db4: 0x3b70, 0x0db5: 0x0015, + 0x0db6: 0x3b73, 0x0db7: 0x3b76, 0x0db8: 0x3b79, 0x0db9: 0x056f, 0x0dba: 0x0575, 0x0dbb: 0x0578, + 0x0dbc: 0x3b7c, 0x0dbd: 0x3b7f, 0x0dbe: 0x3b82, 0x0dbf: 0x056c, // Block 0x37, offset 0xdc0 - 0x0dc0: 0x0001, 0x0dc1: 0x0001, 0x0dc2: 0x0001, 0x0dc3: 0x0001, 0x0dc4: 0x0001, 0x0dc5: 0x0001, - 0x0dc6: 0x0001, 0x0dc7: 0x0001, 0x0dc8: 0x0001, 0x0dc9: 0x0001, 0x0dca: 0x0001, - 0x0dd1: 0x1436, - 0x0dd7: 0x143a, - 0x0de4: 0x143e, 0x0de5: 0x1440, 0x0de6: 0x1443, - 0x0def: 0x0001, - 0x0df3: 0x1447, 0x0df4: 0x144e, - 0x0df6: 0x1458, 0x0df7: 0x145f, - 0x0dfc: 0x1469, 0x0dfe: 0x146c, + 0x0dc0: 0x0979, 0x0dc1: 0x3b85, 0x0dc2: 0x3b88, 0x0dc3: 0x3b8b, 0x0dc4: 0x057e, 0x0dc5: 0x055c, + 0x0dc6: 0x0572, 0x0dc7: 0x056c, 0x0dc8: 0x0575, 0x0dc9: 0x056f, 0x0dca: 0x3b8f, 0x0dcb: 0x3b92, + 0x0dce: 0x1486, 0x0dcf: 0x001c, 0x0dd0: 0x000d, 0x0dd1: 0x000f, + 0x0dd2: 0x1488, 0x0dd3: 0x148a, 0x0dd4: 0x148c, 0x0dd5: 0x148e, 0x0dd6: 0x1490, 0x0dd7: 0x1492, + 0x0dd8: 0x1486, 0x0dd9: 0x001c, 0x0dda: 0x000d, 0x0ddb: 0x000f, 0x0ddc: 0x1488, 0x0ddd: 0x148a, + 0x0dde: 0x148c, 0x0ddf: 0x148e, 0x0de0: 0x1490, 0x0de1: 0x1492, 0x0de2: 0x1486, 0x0de3: 0x001c, + 0x0de4: 0x000d, 0x0de5: 0x000f, 0x0de6: 0x1488, 0x0de7: 0x148a, 0x0de8: 0x148c, 0x0de9: 0x148e, + 0x0dea: 0x1490, 0x0deb: 0x1492, 0x0dec: 0x1486, 0x0ded: 0x001c, 0x0dee: 0x000d, 0x0def: 0x000f, + 0x0df0: 0x1488, 0x0df1: 0x148a, 0x0df2: 0x148c, 0x0df3: 0x148e, 0x0df4: 0x1490, 0x0df5: 0x1492, + 0x0df6: 0x1486, 0x0df7: 0x001c, 0x0df8: 0x000d, 0x0df9: 0x000f, 0x0dfa: 0x1488, 0x0dfb: 0x148a, + 0x0dfc: 0x148c, 0x0dfd: 0x148e, 0x0dfe: 0x1490, 0x0dff: 0x1492, // Block 0x38, offset 0xe00 - 0x0e07: 0x1470, 0x0e08: 0x1473, 0x0e09: 0x1476, - 0x0e17: 0x1479, - 0x0e1f: 0x0001, - 0x0e30: 0x1486, 0x0e31: 0x097c, 0x0e34: 0x1488, 0x0e35: 0x148a, - 0x0e36: 0x148c, 0x0e37: 0x148e, 0x0e38: 0x1490, 0x0e39: 0x1492, 0x0e3a: 0x1494, 0x0e3b: 0x1496, - 0x0e3c: 0x149a, 0x0e3d: 0x149c, 0x0e3e: 0x149e, 0x0e3f: 0x14a0, + 0x0e00: 0x3b95, 0x0e01: 0x3b98, 0x0e02: 0x3b9b, 0x0e03: 0x3b9e, 0x0e04: 0x3ba1, 0x0e05: 0x3ba4, + 0x0e06: 0x3ba7, 0x0e07: 0x3baa, 0x0e08: 0x3bad, 0x0e09: 0x3bb0, 0x0e0a: 0x3bb3, + 0x0e10: 0x3bb6, 0x0e11: 0x3bba, + 0x0e12: 0x3bbe, 0x0e13: 0x3bc2, 0x0e14: 0x3bc6, 0x0e15: 0x3bca, 0x0e16: 0x3bce, 0x0e17: 0x3bd2, + 0x0e18: 0x3bd6, 0x0e19: 0x3bda, 0x0e1a: 0x3bde, 0x0e1b: 0x3be2, 0x0e1c: 0x3be6, 0x0e1d: 0x3bea, + 0x0e1e: 0x3bee, 0x0e1f: 0x3bf2, 0x0e20: 0x3bf6, 0x0e21: 0x3bfa, 0x0e22: 0x3bfe, 0x0e23: 0x3c02, + 0x0e24: 0x3c06, 0x0e25: 0x3c0a, 0x0e26: 0x3c0e, 0x0e27: 0x3c12, 0x0e28: 0x3c16, 0x0e29: 0x3c1a, + 0x0e2a: 0x3c1e, 0x0e2b: 0x14ad, 0x0e2c: 0x092b, 0x0e2d: 0x3c26, 0x0e2e: 0x3c29, + 0x0e30: 0x0906, 0x0e31: 0x090b, 0x0e32: 0x14ad, 0x0e33: 0x090d, 0x0e34: 0x090f, 0x0e35: 0x14d9, + 0x0e36: 0x0914, 0x0e37: 0x0916, 0x0e38: 0x0918, 0x0e39: 0x091a, 0x0e3a: 0x091c, 0x0e3b: 0x091e, + 0x0e3c: 0x0920, 0x0e3d: 0x0922, 0x0e3e: 0x0924, 0x0e3f: 0x0929, // Block 0x39, offset 0xe40 - 0x0e40: 0x1486, 0x0e41: 0x001c, 0x0e42: 0x000d, 0x0e43: 0x000f, 0x0e44: 0x1488, 0x0e45: 0x148a, - 0x0e46: 0x148c, 0x0e47: 0x148e, 0x0e48: 0x1490, 0x0e49: 0x1492, 0x0e4a: 0x1494, 0x0e4b: 0x1496, - 0x0e4c: 0x149a, 0x0e4d: 0x149c, 0x0e4e: 0x149e, 0x0e50: 0x0007, 0x0e51: 0x0941, - 0x0e52: 0x001e, 0x0e53: 0x04c7, 0x0e54: 0x0943, 0x0e55: 0x0494, 0x0e56: 0x094e, 0x0e57: 0x04c5, - 0x0e58: 0x0950, 0x0e59: 0x14a0, 0x0e5a: 0x0960, 0x0e5b: 0x02c8, 0x0e5c: 0x0962, - 0x0e68: 0x14a2, + 0x0e40: 0x3c3f, 0x0e41: 0x3c46, 0x0e42: 0x2291, + 0x0e50: 0x1922, 0x0e51: 0x3c4d, + 0x0e52: 0x3c51, 0x0e53: 0x1cb3, 0x0e54: 0x183e, 0x0e55: 0x3c55, 0x0e56: 0x3c59, 0x0e57: 0x1ed0, + 0x0e58: 0x3c5d, 0x0e59: 0x3c61, 0x0e5a: 0x3c65, 0x0e5b: 0x2d49, 0x0e5c: 0x3c69, 0x0e5d: 0x3c6d, + 0x0e5e: 0x3c71, 0x0e5f: 0x3c75, 0x0e60: 0x3c79, 0x0e61: 0x3c7d, 0x0e62: 0x19b2, 0x0e63: 0x3c81, + 0x0e64: 0x3c85, 0x0e65: 0x3c89, 0x0e66: 0x3c8d, 0x0e67: 0x3c91, 0x0e68: 0x3c95, 0x0e69: 0x1826, + 0x0e6a: 0x1eb0, 0x0e6b: 0x3c99, 0x0e6c: 0x21c7, 0x0e6d: 0x1ebc, 0x0e6e: 0x21cb, 0x0e6f: 0x3c9d, + 0x0e70: 0x1a92, 0x0e71: 0x3ca1, 0x0e72: 0x3ca5, 0x0e73: 0x3ca9, 0x0e74: 0x3cad, 0x0e75: 0x3cb1, + 0x0e76: 0x2183, 0x0e77: 0x194a, 0x0e78: 0x3cb5, 0x0e79: 0x3cb9, 0x0e7a: 0x3cbd, // Block 0x3a, offset 0xe80 - 0x0e80: 0x14a5, 0x0e81: 0x14a9, 0x0e82: 0x14ad, 0x0e83: 0x14af, 0x0e85: 0x14b3, - 0x0e86: 0x14b7, 0x0e87: 0x14bb, 0x0e89: 0x14be, 0x0e8a: 0x094c, 0x0e8b: 0x0916, - 0x0e8c: 0x0916, 0x0e8d: 0x0916, 0x0e8e: 0x0494, 0x0e8f: 0x14c2, 0x0e90: 0x0918, 0x0e91: 0x0918, - 0x0e92: 0x091e, 0x0e93: 0x04c5, 0x0e95: 0x0922, 0x0e96: 0x14c5, - 0x0e99: 0x0929, 0x0e9a: 0x14c8, 0x0e9b: 0x092b, 0x0e9c: 0x092b, 0x0e9d: 0x092b, - 0x0ea0: 0x14ca, 0x0ea1: 0x14cd, 0x0ea2: 0x14d1, - 0x0ea4: 0x14d4, 0x0ea6: 0x14d6, 0x0ea8: 0x14d4, - 0x0eaa: 0x091c, 0x0eab: 0x0046, 0x0eac: 0x090b, 0x0ead: 0x14ad, 0x0eaf: 0x0941, - 0x0eb0: 0x090f, 0x0eb1: 0x14d9, 0x0eb3: 0x0920, 0x0eb4: 0x001e, 0x0eb5: 0x14db, - 0x0eb6: 0x14de, 0x0eb7: 0x14e1, 0x0eb8: 0x14e4, 0x0eb9: 0x097c, 0x0ebb: 0x14e7, - 0x0ebc: 0x056f, 0x0ebd: 0x0973, 0x0ebe: 0x14eb, 0x0ebf: 0x14ee, + 0x0e80: 0x3d23, 0x0e81: 0x3d27, 0x0e82: 0x3d2b, 0x0e83: 0x3d2f, 0x0e84: 0x3d34, 0x0e85: 0x2eb5, + 0x0e86: 0x3d38, 0x0e87: 0x3d3c, 0x0e88: 0x3d40, 0x0e89: 0x3d44, 0x0e8a: 0x2eb9, 0x0e8b: 0x3d48, + 0x0e8c: 0x3d4c, 0x0e8d: 0x3d50, 0x0e8e: 0x2ebd, 0x0e8f: 0x3d55, 0x0e90: 0x3d59, 0x0e91: 0x3d5d, + 0x0e92: 0x3d61, 0x0e93: 0x3d66, 0x0e94: 0x3d6a, 0x0e95: 0x3c71, 0x0e96: 0x3d6e, 0x0e97: 0x3d73, + 0x0e98: 0x3d77, 0x0e99: 0x3d7b, 0x0e9a: 0x3d7f, 0x0e9b: 0x2f9a, 0x0e9c: 0x3d83, 0x0e9d: 0x1866, + 0x0e9e: 0x3d88, 0x0e9f: 0x3d8c, 0x0ea0: 0x3d90, 0x0ea1: 0x3d94, 0x0ea2: 0x3cb9, 0x0ea3: 0x3d98, + 0x0ea4: 0x3d9c, 0x0ea5: 0x2fae, 0x0ea6: 0x2ec1, 0x0ea7: 0x2ec5, 0x0ea8: 0x2fb2, 0x0ea9: 0x3da0, + 0x0eaa: 0x3da4, 0x0eab: 0x2bf1, 0x0eac: 0x3da8, 0x0ead: 0x2ec9, 0x0eae: 0x3dac, 0x0eaf: 0x3db0, + 0x0eb0: 0x3db4, 0x0eb1: 0x3db8, 0x0eb2: 0x3db8, 0x0eb3: 0x3db8, 0x0eb4: 0x3dbc, 0x0eb5: 0x3dc1, + 0x0eb6: 0x3dc5, 0x0eb7: 0x3dc9, 0x0eb8: 0x3dcd, 0x0eb9: 0x3dd2, 0x0eba: 0x3dd6, 0x0ebb: 0x3dda, + 0x0ebc: 0x3dde, 0x0ebd: 0x3de2, 0x0ebe: 0x3de6, 0x0ebf: 0x3dea, // Block 0x3b, offset 0xec0 - 0x0ec0: 0x14f1, 0x0ec5: 0x090d, - 0x0ec6: 0x093f, 0x0ec7: 0x0941, 0x0ec8: 0x097c, 0x0ec9: 0x0499, - 0x0ed0: 0x14f5, 0x0ed1: 0x14fb, - 0x0ed2: 0x1501, 0x0ed3: 0x1508, 0x0ed4: 0x150e, 0x0ed5: 0x1514, 0x0ed6: 0x151a, 0x0ed7: 0x1520, - 0x0ed8: 0x1526, 0x0ed9: 0x152c, 0x0eda: 0x1532, 0x0edb: 0x1538, 0x0edc: 0x153e, 0x0edd: 0x1544, - 0x0ede: 0x154a, 0x0edf: 0x1550, 0x0ee0: 0x0918, 0x0ee1: 0x1555, 0x0ee2: 0x1558, 0x0ee3: 0x155c, - 0x0ee4: 0x155f, 0x0ee5: 0x1561, 0x0ee6: 0x1564, 0x0ee7: 0x1568, 0x0ee8: 0x156d, 0x0ee9: 0x1570, - 0x0eea: 0x1572, 0x0eeb: 0x1575, 0x0eec: 0x091e, 0x0eed: 0x14ad, 0x0eee: 0x090d, 0x0eef: 0x0920, - 0x0ef0: 0x097c, 0x0ef1: 0x1579, 0x0ef2: 0x157c, 0x0ef3: 0x1580, 0x0ef4: 0x096d, 0x0ef5: 0x1583, - 0x0ef6: 0x1586, 0x0ef7: 0x158a, 0x0ef8: 0x158f, 0x0ef9: 0x04c7, 0x0efa: 0x1592, 0x0efb: 0x1595, - 0x0efc: 0x04c5, 0x0efd: 0x0984, 0x0efe: 0x093f, 0x0eff: 0x0950, + 0x0ec0: 0x3dee, 0x0ec1: 0x3df2, 0x0ec2: 0x3df6, 0x0ec3: 0x3dfa, 0x0ec4: 0x3dfe, 0x0ec5: 0x3e02, + 0x0ec6: 0x3e02, 0x0ec7: 0x2fba, 0x0ec8: 0x3e06, 0x0ec9: 0x3e0a, 0x0eca: 0x3e0e, 0x0ecb: 0x3e12, + 0x0ecc: 0x2ed1, 0x0ecd: 0x3e16, 0x0ece: 0x3e1a, 0x0ecf: 0x3e1e, 0x0ed0: 0x2e39, 0x0ed1: 0x3e22, + 0x0ed2: 0x3e26, 0x0ed3: 0x3e2a, 0x0ed4: 0x3e2e, 0x0ed5: 0x3e32, 0x0ed6: 0x3e36, 0x0ed7: 0x3e3a, + 0x0ed8: 0x3e3e, 0x0ed9: 0x3e42, 0x0eda: 0x3e47, 0x0edb: 0x3e4b, 0x0edc: 0x3e4f, 0x0edd: 0x3c55, + 0x0ede: 0x3e53, 0x0edf: 0x3e57, 0x0ee0: 0x3e5b, 0x0ee1: 0x3e60, 0x0ee2: 0x3e65, 0x0ee3: 0x3e69, + 0x0ee4: 0x3e6d, 0x0ee5: 0x3e71, 0x0ee6: 0x3e75, 0x0ee7: 0x3e79, 0x0ee8: 0x3e7d, 0x0ee9: 0x3e81, + 0x0eea: 0x3e85, 0x0eeb: 0x3e85, 0x0eec: 0x3e89, 0x0eed: 0x3e8e, 0x0eee: 0x3e92, 0x0eef: 0x2be1, + 0x0ef0: 0x3e96, 0x0ef1: 0x3e9a, 0x0ef2: 0x3e9f, 0x0ef3: 0x3ea3, 0x0ef4: 0x3ea7, 0x0ef5: 0x18ce, + 0x0ef6: 0x3eab, 0x0ef7: 0x3eaf, 0x0ef8: 0x18d6, 0x0ef9: 0x3eb3, 0x0efa: 0x3eb7, 0x0efb: 0x3ebb, + 0x0efc: 0x3ec0, 0x0efd: 0x3ec4, 0x0efe: 0x3ec9, 0x0eff: 0x3ecd, // Block 0x3c, offset 0xf00 - 0x0f09: 0x1599, - 0x0f1a: 0x159f, 0x0f1b: 0x15a5, - 0x0f2e: 0x15ab, + 0x0f00: 0x3ed1, 0x0f01: 0x3ed5, 0x0f02: 0x3ed9, 0x0f03: 0x3edd, 0x0f04: 0x3ee1, 0x0f05: 0x3ee5, + 0x0f06: 0x3ee9, 0x0f07: 0x3eed, 0x0f08: 0x3ef1, 0x0f09: 0x3ef5, 0x0f0a: 0x3efa, 0x0f0b: 0x3efe, + 0x0f0c: 0x3f02, 0x0f0d: 0x3f06, 0x0f0e: 0x2b11, 0x0f0f: 0x3f0a, 0x0f10: 0x18fe, 0x0f11: 0x3f0f, + 0x0f12: 0x3f0f, 0x0f13: 0x3f14, 0x0f14: 0x3f18, 0x0f15: 0x3f18, 0x0f16: 0x3f1c, 0x0f17: 0x3f20, + 0x0f18: 0x3f25, 0x0f19: 0x3f2a, 0x0f1a: 0x3f2e, 0x0f1b: 0x3f32, 0x0f1c: 0x3f36, 0x0f1d: 0x3f3a, + 0x0f1e: 0x3f3e, 0x0f1f: 0x3f42, 0x0f20: 0x3f46, 0x0f21: 0x3f4a, 0x0f22: 0x3f4e, 0x0f23: 0x2ee5, + 0x0f24: 0x3f52, 0x0f25: 0x3f57, 0x0f26: 0x3f5b, 0x0f27: 0x3f5f, 0x0f28: 0x2fea, 0x0f29: 0x3f5f, + 0x0f2a: 0x3f63, 0x0f2b: 0x2eed, 0x0f2c: 0x3f67, 0x0f2d: 0x3f6b, 0x0f2e: 0x3f6f, 0x0f2f: 0x3f73, + 0x0f30: 0x2ef1, 0x0f31: 0x2aa5, 0x0f32: 0x3f77, 0x0f33: 0x3f7b, 0x0f34: 0x3f7f, 0x0f35: 0x3f83, + 0x0f36: 0x3f87, 0x0f37: 0x3f8b, 0x0f38: 0x3f8f, 0x0f39: 0x3f94, 0x0f3a: 0x3f98, 0x0f3b: 0x3f9c, + 0x0f3c: 0x3fa0, 0x0f3d: 0x3fa4, 0x0f3e: 0x3fa8, 0x0f3f: 0x3fad, // Block 0x3d, offset 0xf40 - 0x0f4d: 0x15b1, 0x0f4e: 0x15b7, 0x0f4f: 0x15bd, + 0x0f40: 0x3fb1, 0x0f41: 0x3fb5, 0x0f42: 0x3fb9, 0x0f43: 0x3fbd, 0x0f44: 0x3fc1, 0x0f45: 0x3fc5, + 0x0f46: 0x3fc9, 0x0f47: 0x3fcd, 0x0f48: 0x2ef5, 0x0f49: 0x3fd1, 0x0f4a: 0x3fd5, 0x0f4b: 0x3fda, + 0x0f4c: 0x3fde, 0x0f4d: 0x3fe2, 0x0f4e: 0x3fe6, 0x0f4f: 0x2efd, 0x0f50: 0x3fea, 0x0f51: 0x3fee, + 0x0f52: 0x3ff2, 0x0f53: 0x3ff6, 0x0f54: 0x3ffa, 0x0f55: 0x3ffe, 0x0f56: 0x4002, 0x0f57: 0x4006, + 0x0f58: 0x2b15, 0x0f59: 0x300a, 0x0f5a: 0x400a, 0x0f5b: 0x400e, 0x0f5c: 0x4012, 0x0f5d: 0x4016, + 0x0f5e: 0x401b, 0x0f5f: 0x401f, 0x0f60: 0x4023, 0x0f61: 0x4027, 0x0f62: 0x2f01, 0x0f63: 0x402b, + 0x0f64: 0x4030, 0x0f65: 0x4034, 0x0f66: 0x4038, 0x0f67: 0x30b5, 0x0f68: 0x403c, 0x0f69: 0x4040, + 0x0f6a: 0x4044, 0x0f6b: 0x4048, 0x0f6c: 0x404c, 0x0f6d: 0x4051, 0x0f6e: 0x4055, 0x0f6f: 0x4059, + 0x0f70: 0x405d, 0x0f71: 0x4062, 0x0f72: 0x4066, 0x0f73: 0x406a, 0x0f74: 0x406e, 0x0f75: 0x2c25, + 0x0f76: 0x4072, 0x0f77: 0x4076, 0x0f78: 0x407b, 0x0f79: 0x4080, 0x0f7a: 0x4085, 0x0f7b: 0x4089, + 0x0f7c: 0x408e, 0x0f7d: 0x4092, 0x0f7e: 0x4096, 0x0f7f: 0x409a, // Block 0x3e, offset 0xf80 - 0x0f84: 0x15c3, - 0x0f89: 0x15c9, - 0x0f8c: 0x15cf, - 0x0fa4: 0x15d5, 0x0fa6: 0x15db, - 0x0fac: 0x15e1, 0x0fad: 0x15e8, 0x0faf: 0x15f2, - 0x0fb0: 0x15f9, + 0x0f80: 0x409e, 0x0f81: 0x2f05, 0x0f82: 0x2d71, 0x0f83: 0x40a2, 0x0f84: 0x40a6, 0x0f85: 0x40aa, + 0x0f86: 0x40ae, 0x0f87: 0x40b3, 0x0f88: 0x40b7, 0x0f89: 0x40bb, 0x0f8a: 0x40bf, 0x0f8b: 0x3016, + 0x0f8c: 0x40c3, 0x0f8d: 0x40c7, 0x0f8e: 0x40cc, 0x0f8f: 0x40d0, 0x0f90: 0x40d4, 0x0f91: 0x40d9, + 0x0f92: 0x40de, 0x0f93: 0x40e2, 0x0f94: 0x301a, 0x0f95: 0x40e6, 0x0f96: 0x40ea, 0x0f97: 0x40ee, + 0x0f98: 0x40f2, 0x0f99: 0x40f6, 0x0f9a: 0x40fa, 0x0f9b: 0x40fe, 0x0f9c: 0x4103, 0x0f9d: 0x4107, + 0x0f9e: 0x410c, 0x0f9f: 0x4110, 0x0fa0: 0x4115, 0x0fa1: 0x3022, 0x0fa2: 0x4119, 0x0fa3: 0x411d, + 0x0fa4: 0x4122, 0x0fa5: 0x4126, 0x0fa6: 0x412a, 0x0fa7: 0x412f, 0x0fa8: 0x4134, 0x0fa9: 0x4138, + 0x0faa: 0x413c, 0x0fab: 0x4140, 0x0fac: 0x4144, 0x0fad: 0x4144, 0x0fae: 0x4148, 0x0faf: 0x414c, + 0x0fb0: 0x302a, 0x0fb1: 0x4150, 0x0fb2: 0x4154, 0x0fb3: 0x4158, 0x0fb4: 0x415c, 0x0fb5: 0x4160, + 0x0fb6: 0x4165, 0x0fb7: 0x4169, 0x0fb8: 0x2bed, 0x0fb9: 0x416e, 0x0fba: 0x4173, 0x0fbb: 0x4177, + 0x0fbc: 0x417c, 0x0fbd: 0x4181, 0x0fbe: 0x4186, 0x0fbf: 0x418a, // Block 0x3f, offset 0xfc0 - 0x0fc1: 0x1603, 0x0fc4: 0x1609, - 0x0fc7: 0x160f, 0x0fc9: 0x1615, - 0x0fe0: 0x161b, 0x0fe2: 0x161f, - 0x0fed: 0x1625, 0x0fee: 0x162b, 0x0fef: 0x162f, - 0x0ff0: 0x1633, 0x0ff1: 0x1639, 0x0ff4: 0x163f, 0x0ff5: 0x1645, - 0x0ff8: 0x164b, 0x0ff9: 0x1651, + 0x0fc0: 0x3042, 0x0fc1: 0x418e, 0x0fc2: 0x4193, 0x0fc3: 0x4198, 0x0fc4: 0x419d, 0x0fc5: 0x41a2, + 0x0fc6: 0x41a6, 0x0fc7: 0x41a6, 0x0fc8: 0x3046, 0x0fc9: 0x30bd, 0x0fca: 0x41aa, 0x0fcb: 0x41ae, + 0x0fcc: 0x41b2, 0x0fcd: 0x41b6, 0x0fce: 0x41bb, 0x0fcf: 0x2b59, 0x0fd0: 0x304e, 0x0fd1: 0x41bf, + 0x0fd2: 0x41c3, 0x0fd3: 0x2f2d, 0x0fd4: 0x41c8, 0x0fd5: 0x41cd, 0x0fd6: 0x2e89, 0x0fd7: 0x41d2, + 0x0fd8: 0x41d6, 0x0fd9: 0x2f39, 0x0fda: 0x41da, 0x0fdb: 0x41de, 0x0fdc: 0x41e2, 0x0fdd: 0x41e7, + 0x0fde: 0x41e7, 0x0fdf: 0x41ec, 0x0fe0: 0x41f0, 0x0fe1: 0x41f4, 0x0fe2: 0x41f9, 0x0fe3: 0x41fd, + 0x0fe4: 0x4201, 0x0fe5: 0x4205, 0x0fe6: 0x420a, 0x0fe7: 0x420e, 0x0fe8: 0x4212, 0x0fe9: 0x4216, + 0x0fea: 0x421a, 0x0feb: 0x421e, 0x0fec: 0x4223, 0x0fed: 0x4227, 0x0fee: 0x422b, 0x0fef: 0x422f, + 0x0ff0: 0x4233, 0x0ff1: 0x4237, 0x0ff2: 0x423b, 0x0ff3: 0x4240, 0x0ff4: 0x4245, 0x0ff5: 0x4249, + 0x0ff6: 0x424e, 0x0ff7: 0x4252, 0x0ff8: 0x4257, 0x0ff9: 0x425b, 0x0ffa: 0x2f51, 0x0ffb: 0x425f, + 0x0ffc: 0x4264, 0x0ffd: 0x4269, 0x0ffe: 0x426d, 0x0fff: 0x4272, // Block 0x40, offset 0x1000 - 0x1000: 0x1657, 0x1001: 0x165d, 0x1004: 0x1663, 0x1005: 0x1669, - 0x1008: 0x166f, 0x1009: 0x1675, - 0x102c: 0x167b, 0x102d: 0x1681, 0x102e: 0x1687, 0x102f: 0x168d, + 0x1000: 0x4276, 0x1001: 0x427b, 0x1002: 0x427f, 0x1003: 0x4283, 0x1004: 0x4287, 0x1005: 0x428b, + 0x1006: 0x428f, 0x1007: 0x4293, 0x1008: 0x4298, 0x1009: 0x429d, 0x100a: 0x42a2, 0x100b: 0x3f14, + 0x100c: 0x42a7, 0x100d: 0x42ab, 0x100e: 0x42af, 0x100f: 0x42b3, 0x1010: 0x42b7, 0x1011: 0x42bb, + 0x1012: 0x42bf, 0x1013: 0x42c3, 0x1014: 0x42c7, 0x1015: 0x42cb, 0x1016: 0x42cf, 0x1017: 0x42d3, + 0x1018: 0x2c31, 0x1019: 0x42d8, 0x101a: 0x42dc, 0x101b: 0x42e0, 0x101c: 0x42e4, 0x101d: 0x42e8, + 0x101e: 0x42ec, 0x101f: 0x2f5d, 0x1020: 0x42f0, 0x1021: 0x42f4, 0x1022: 0x42f8, 0x1023: 0x42fc, + 0x1024: 0x4300, 0x1025: 0x4305, 0x1026: 0x430a, 0x1027: 0x430f, 0x1028: 0x4313, 0x1029: 0x4317, + 0x102a: 0x431b, 0x102b: 0x431f, 0x102c: 0x4324, 0x102d: 0x4328, 0x102e: 0x432d, 0x102f: 0x4331, + 0x1030: 0x4335, 0x1031: 0x433a, 0x1032: 0x433f, 0x1033: 0x4343, 0x1034: 0x2b45, 0x1035: 0x4347, + 0x1036: 0x434b, 0x1037: 0x434f, 0x1038: 0x4353, 0x1039: 0x4357, 0x103a: 0x435b, 0x103b: 0x306a, + 0x103c: 0x435f, 0x103d: 0x4363, 0x103e: 0x4367, 0x103f: 0x436b, // Block 0x41, offset 0x1040 - 0x1060: 0x1693, 0x1061: 0x1699, 0x1062: 0x169f, 0x1063: 0x16a5, - 0x106a: 0x16ab, 0x106b: 0x16b1, 0x106c: 0x16b7, 0x106d: 0x16bd, - // Block 0x42, offset 0x1080 - 0x10a9: 0x16c3, - 0x10aa: 0x16c7, - // Block 0x43, offset 0x10c0 - 0x10e0: 0x001c, 0x10e1: 0x000d, 0x10e2: 0x000f, 0x10e3: 0x1488, - 0x10e4: 0x148a, 0x10e5: 0x148c, 0x10e6: 0x148e, 0x10e7: 0x1490, 0x10e8: 0x1492, 0x10e9: 0x16cb, - 0x10ea: 0x16ce, 0x10eb: 0x16d1, 0x10ec: 0x16d4, 0x10ed: 0x16d7, 0x10ee: 0x16da, 0x10ef: 0x16dd, - 0x10f0: 0x16e0, 0x10f1: 0x16e3, 0x10f2: 0x16e6, 0x10f3: 0x16e9, 0x10f4: 0x16ec, 0x10f5: 0x16f0, - 0x10f6: 0x16f4, 0x10f7: 0x16f8, 0x10f8: 0x16fc, 0x10f9: 0x1700, 0x10fa: 0x1704, 0x10fb: 0x1708, - 0x10fc: 0x170c, 0x10fd: 0x1710, 0x10fe: 0x1715, 0x10ff: 0x171a, - // Block 0x44, offset 0x1100 - 0x1100: 0x171f, 0x1101: 0x1724, 0x1102: 0x1729, 0x1103: 0x172e, 0x1104: 0x1733, 0x1105: 0x1738, - 0x1106: 0x173d, 0x1107: 0x1742, 0x1108: 0x1747, 0x1109: 0x174a, 0x110a: 0x174d, 0x110b: 0x1750, - 0x110c: 0x1753, 0x110d: 0x1756, 0x110e: 0x1759, 0x110f: 0x175c, 0x1110: 0x175f, 0x1111: 0x1762, - 0x1112: 0x1766, 0x1113: 0x176a, 0x1114: 0x176e, 0x1115: 0x1772, 0x1116: 0x1776, 0x1117: 0x177a, - 0x1118: 0x177e, 0x1119: 0x1782, 0x111a: 0x1786, 0x111b: 0x178a, 0x111c: 0x178e, 0x111d: 0x1792, - 0x111e: 0x1796, 0x111f: 0x179a, 0x1120: 0x179e, 0x1121: 0x17a2, 0x1122: 0x17a6, 0x1123: 0x17aa, - 0x1124: 0x17ae, 0x1125: 0x17b2, 0x1126: 0x17b6, 0x1127: 0x17ba, 0x1128: 0x17be, 0x1129: 0x17c2, - 0x112a: 0x17c6, 0x112b: 0x17ca, 0x112c: 0x17ce, 0x112d: 0x17d2, 0x112e: 0x17d6, 0x112f: 0x17da, - 0x1130: 0x17de, 0x1131: 0x17e2, 0x1132: 0x17e6, 0x1133: 0x17ea, 0x1134: 0x17ee, 0x1135: 0x17f2, - 0x1136: 0x0906, 0x1137: 0x090b, 0x1138: 0x14ad, 0x1139: 0x090d, 0x113a: 0x090f, 0x113b: 0x14d9, - 0x113c: 0x0914, 0x113d: 0x0916, 0x113e: 0x0918, 0x113f: 0x091a, - // Block 0x45, offset 0x1140 - 0x1140: 0x091c, 0x1141: 0x091e, 0x1142: 0x0920, 0x1143: 0x0922, 0x1144: 0x0924, 0x1145: 0x0929, - 0x1146: 0x14c8, 0x1147: 0x092b, 0x1148: 0x17f6, 0x1149: 0x092d, 0x114a: 0x092f, 0x114b: 0x155f, - 0x114c: 0x0931, 0x114d: 0x1570, 0x114e: 0x17f8, 0x114f: 0x14d4, 0x1150: 0x0007, 0x1151: 0x093d, - 0x1152: 0x0984, 0x1153: 0x093f, 0x1154: 0x0941, 0x1155: 0x098c, 0x1156: 0x094c, 0x1157: 0x0494, - 0x1158: 0x097c, 0x1159: 0x0499, 0x115a: 0x094e, 0x115b: 0x04c5, 0x115c: 0x0950, 0x115d: 0x14a0, - 0x115e: 0x001e, 0x115f: 0x0960, 0x1160: 0x17fa, 0x1161: 0x049b, 0x1162: 0x02c8, 0x1163: 0x0962, - 0x1164: 0x0964, 0x1165: 0x096d, 0x1166: 0x04a6, 0x1167: 0x04c7, 0x1168: 0x04a8, 0x1169: 0x09df, - 0x116a: 0x1486, - // Block 0x46, offset 0x1180 - 0x118c: 0x17fc, - // Block 0x47, offset 0x11c0 - 0x11f4: 0x1809, 0x11f5: 0x180d, - 0x11f6: 0x1810, - // Block 0x48, offset 0x1200 - 0x121c: 0x1814, - // Block 0x49, offset 0x1240 - 0x127c: 0x0499, 0x127d: 0x155f, - // Block 0x4a, offset 0x1280 - 0x12af: 0x181a, - // Block 0x4b, offset 0x12c0 - 0x12df: 0x181e, - // Block 0x4c, offset 0x1300 - 0x1333: 0x1822, - // Block 0x4d, offset 0x1340 - 0x1340: 0x1826, 0x1341: 0x182a, 0x1342: 0x182e, 0x1343: 0x1832, 0x1344: 0x1836, 0x1345: 0x183a, - 0x1346: 0x183e, 0x1347: 0x1842, 0x1348: 0x1846, 0x1349: 0x184a, 0x134a: 0x184e, 0x134b: 0x1852, - 0x134c: 0x1856, 0x134d: 0x185a, 0x134e: 0x185e, 0x134f: 0x1862, 0x1350: 0x1866, 0x1351: 0x186a, - 0x1352: 0x186e, 0x1353: 0x1872, 0x1354: 0x1876, 0x1355: 0x187a, 0x1356: 0x187e, 0x1357: 0x1882, - 0x1358: 0x1886, 0x1359: 0x188a, 0x135a: 0x188e, 0x135b: 0x1892, 0x135c: 0x1896, 0x135d: 0x189a, - 0x135e: 0x189e, 0x135f: 0x18a2, 0x1360: 0x18a6, 0x1361: 0x18aa, 0x1362: 0x18ae, 0x1363: 0x18b2, - 0x1364: 0x18b6, 0x1365: 0x18ba, 0x1366: 0x18be, 0x1367: 0x18c2, 0x1368: 0x18c6, 0x1369: 0x18ca, - 0x136a: 0x18ce, 0x136b: 0x18d2, 0x136c: 0x18d6, 0x136d: 0x18da, 0x136e: 0x18de, 0x136f: 0x18e2, - 0x1370: 0x18e6, 0x1371: 0x18ea, 0x1372: 0x18ee, 0x1373: 0x18f2, 0x1374: 0x18f6, 0x1375: 0x18fa, - 0x1376: 0x18fe, 0x1377: 0x1902, 0x1378: 0x1906, 0x1379: 0x190a, 0x137a: 0x190e, 0x137b: 0x1912, - 0x137c: 0x1916, 0x137d: 0x191a, 0x137e: 0x191e, 0x137f: 0x1922, - // Block 0x4e, offset 0x1380 - 0x1380: 0x1926, 0x1381: 0x192a, 0x1382: 0x192e, 0x1383: 0x1932, 0x1384: 0x1936, 0x1385: 0x193a, - 0x1386: 0x193e, 0x1387: 0x1942, 0x1388: 0x1946, 0x1389: 0x194a, 0x138a: 0x194e, 0x138b: 0x1952, - 0x138c: 0x1956, 0x138d: 0x195a, 0x138e: 0x195e, 0x138f: 0x1962, 0x1390: 0x1966, 0x1391: 0x196a, - 0x1392: 0x196e, 0x1393: 0x1972, 0x1394: 0x1976, 0x1395: 0x197a, 0x1396: 0x197e, 0x1397: 0x1982, - 0x1398: 0x1986, 0x1399: 0x198a, 0x139a: 0x198e, 0x139b: 0x1992, 0x139c: 0x1996, 0x139d: 0x199a, - 0x139e: 0x199e, 0x139f: 0x19a2, 0x13a0: 0x19a6, 0x13a1: 0x19aa, 0x13a2: 0x19ae, 0x13a3: 0x19b2, - 0x13a4: 0x19b6, 0x13a5: 0x19ba, 0x13a6: 0x19be, 0x13a7: 0x19c2, 0x13a8: 0x19c6, 0x13a9: 0x19ca, - 0x13aa: 0x19ce, 0x13ab: 0x19d2, 0x13ac: 0x19d6, 0x13ad: 0x19da, 0x13ae: 0x19de, 0x13af: 0x19e2, - 0x13b0: 0x19e6, 0x13b1: 0x19ea, 0x13b2: 0x19ee, 0x13b3: 0x19f2, 0x13b4: 0x19f6, 0x13b5: 0x19fa, - 0x13b6: 0x19fe, 0x13b7: 0x1a02, 0x13b8: 0x1a06, 0x13b9: 0x1a0a, 0x13ba: 0x1a0e, 0x13bb: 0x1a12, - 0x13bc: 0x1a16, 0x13bd: 0x1a1a, 0x13be: 0x1a1e, 0x13bf: 0x1a22, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x1a26, 0x13c1: 0x1a2a, 0x13c2: 0x1a2e, 0x13c3: 0x1a32, 0x13c4: 0x1a36, 0x13c5: 0x1a3a, - 0x13c6: 0x1a3e, 0x13c7: 0x1a42, 0x13c8: 0x1a46, 0x13c9: 0x1a4a, 0x13ca: 0x1a4e, 0x13cb: 0x1a52, - 0x13cc: 0x1a56, 0x13cd: 0x1a5a, 0x13ce: 0x1a5e, 0x13cf: 0x1a62, 0x13d0: 0x1a66, 0x13d1: 0x1a6a, - 0x13d2: 0x1a6e, 0x13d3: 0x1a72, 0x13d4: 0x1a76, 0x13d5: 0x1a7a, 0x13d6: 0x1a7e, 0x13d7: 0x1a82, - 0x13d8: 0x1a86, 0x13d9: 0x1a8a, 0x13da: 0x1a8e, 0x13db: 0x1a92, 0x13dc: 0x1a96, 0x13dd: 0x1a9a, - 0x13de: 0x1a9e, 0x13df: 0x1aa2, 0x13e0: 0x1aa6, 0x13e1: 0x1aaa, 0x13e2: 0x1aae, 0x13e3: 0x1ab2, - 0x13e4: 0x1ab6, 0x13e5: 0x1aba, 0x13e6: 0x1abe, 0x13e7: 0x1ac2, 0x13e8: 0x1ac6, 0x13e9: 0x1aca, - 0x13ea: 0x1ace, 0x13eb: 0x1ad2, 0x13ec: 0x1ad6, 0x13ed: 0x1ada, 0x13ee: 0x1ade, 0x13ef: 0x1ae2, - 0x13f0: 0x1ae6, 0x13f1: 0x1aea, 0x13f2: 0x1aee, 0x13f3: 0x1af2, 0x13f4: 0x1af6, 0x13f5: 0x1afa, - 0x13f6: 0x1afe, 0x13f7: 0x1b02, 0x13f8: 0x1b06, 0x13f9: 0x1b0a, 0x13fa: 0x1b0e, 0x13fb: 0x1b12, - 0x13fc: 0x1b16, 0x13fd: 0x1b1a, 0x13fe: 0x1b1e, 0x13ff: 0x1b22, - // Block 0x50, offset 0x1400 - 0x1400: 0x1b26, 0x1401: 0x1b2a, 0x1402: 0x1b2e, 0x1403: 0x1b32, 0x1404: 0x1b36, 0x1405: 0x1b3a, - 0x1406: 0x1b3e, 0x1407: 0x1b42, 0x1408: 0x1b46, 0x1409: 0x1b4a, 0x140a: 0x1b4e, 0x140b: 0x1b52, - 0x140c: 0x1b56, 0x140d: 0x1b5a, 0x140e: 0x1b5e, 0x140f: 0x1b62, 0x1410: 0x1b66, 0x1411: 0x1b6a, - 0x1412: 0x1b6e, 0x1413: 0x1b72, 0x1414: 0x1b76, 0x1415: 0x1b7a, - // Block 0x51, offset 0x1440 - 0x1440: 0x0001, - 0x1476: 0x1b7e, 0x1478: 0x1882, 0x1479: 0x1b82, 0x147a: 0x1b86, - // Block 0x52, offset 0x1480 - 0x148c: 0x1b8a, 0x148e: 0x1b91, 0x1490: 0x1b98, - 0x1492: 0x1b9f, 0x1494: 0x1ba6, 0x1496: 0x1bad, - 0x1498: 0x1bb4, 0x149a: 0x1bbb, 0x149c: 0x1bc2, - 0x149e: 0x1bc9, 0x14a0: 0x1bd0, 0x14a2: 0x1bd7, - 0x14a5: 0x1bde, 0x14a7: 0x1be5, 0x14a9: 0x1bec, - 0x14b0: 0x1bf3, 0x14b1: 0x1bfa, 0x14b3: 0x1c01, 0x14b4: 0x1c08, - 0x14b6: 0x1c0f, 0x14b7: 0x1c16, 0x14b9: 0x1c1d, 0x14ba: 0x1c24, - 0x14bc: 0x1c2b, 0x14bd: 0x1c32, - // Block 0x53, offset 0x14c0 - 0x14d4: 0x1c39, - 0x14db: 0x1c40, 0x14dc: 0x1c45, - 0x14de: 0x1c4a, 0x14df: 0x1c51, - 0x14ec: 0x1c58, 0x14ee: 0x1c5f, - 0x14f0: 0x1c66, 0x14f2: 0x1c6d, 0x14f4: 0x1c74, - 0x14f6: 0x1c7b, 0x14f8: 0x1c82, 0x14fa: 0x1c89, - 0x14fc: 0x1c90, 0x14fe: 0x1c97, - // Block 0x54, offset 0x1500 - 0x1500: 0x1c9e, 0x1502: 0x1ca5, 0x1505: 0x1cac, - 0x1507: 0x1cb3, 0x1509: 0x1cba, - 0x1510: 0x1cc1, 0x1511: 0x1cc8, - 0x1513: 0x1ccf, 0x1514: 0x1cd6, 0x1516: 0x1cdd, 0x1517: 0x1ce4, - 0x1519: 0x1ceb, 0x151a: 0x1cf2, 0x151c: 0x1cf9, 0x151d: 0x1d00, - 0x1534: 0x1d07, - 0x1537: 0x1d0e, 0x1538: 0x1d15, 0x1539: 0x1d1c, 0x153a: 0x1d23, - 0x153e: 0x1d2a, 0x153f: 0x1d31, - // Block 0x55, offset 0x1540 - 0x1571: 0x1d38, 0x1572: 0x1d3c, 0x1573: 0x1d40, 0x1574: 0x1d44, 0x1575: 0x1d48, - 0x1576: 0x1d4c, 0x1577: 0x1d50, 0x1578: 0x1d54, 0x1579: 0x1d58, 0x157a: 0x1d5c, 0x157b: 0x1d60, - 0x157c: 0x1d64, 0x157d: 0x1d68, 0x157e: 0x1d6c, 0x157f: 0x1d70, - // Block 0x56, offset 0x1580 - 0x1580: 0x1d74, 0x1581: 0x1d78, 0x1582: 0x1d7c, 0x1583: 0x1d80, 0x1584: 0x1d84, 0x1585: 0x1d88, - 0x1586: 0x1d8c, 0x1587: 0x1d90, 0x1588: 0x1d94, 0x1589: 0x1d98, 0x158a: 0x1d9c, 0x158b: 0x1da0, - 0x158c: 0x1da4, 0x158d: 0x1da8, 0x158e: 0x1dac, 0x158f: 0x1db0, 0x1590: 0x1db4, 0x1591: 0x1db8, - 0x1592: 0x1dbc, 0x1593: 0x1dc0, 0x1594: 0x1dc4, 0x1595: 0x1dc8, 0x1596: 0x1dcc, 0x1597: 0x1dd0, - 0x1598: 0x1dd4, 0x1599: 0x1dd8, 0x159a: 0x1ddc, 0x159b: 0x1de0, 0x159c: 0x1de4, 0x159d: 0x1de8, - 0x159e: 0x1dec, 0x159f: 0x1df0, 0x15a0: 0x1df4, 0x15a1: 0x1df8, 0x15a2: 0x1dfc, 0x15a3: 0x1e00, - 0x15a4: 0x1e04, 0x15a5: 0x1e08, 0x15a6: 0x1e0c, 0x15a7: 0x1e10, 0x15a8: 0x1e14, 0x15a9: 0x1e18, - 0x15aa: 0x1e1c, 0x15ab: 0x1e20, 0x15ac: 0x1e24, 0x15ad: 0x1e28, 0x15ae: 0x1e2c, 0x15af: 0x1e30, - 0x15b0: 0x1e34, 0x15b1: 0x1e38, 0x15b2: 0x1e3c, 0x15b3: 0x1e40, 0x15b4: 0x1e44, 0x15b5: 0x1e48, - 0x15b6: 0x1e4c, 0x15b7: 0x1e50, 0x15b8: 0x1e54, 0x15b9: 0x1e58, 0x15ba: 0x1e5c, 0x15bb: 0x1e60, - 0x15bc: 0x1e64, 0x15bd: 0x1e68, 0x15be: 0x1e6c, 0x15bf: 0x1e70, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x1e74, 0x15c1: 0x1e78, 0x15c2: 0x1e7c, 0x15c3: 0x1e80, 0x15c4: 0x1e84, 0x15c5: 0x1e88, - 0x15c6: 0x1e8c, 0x15c7: 0x1e90, 0x15c8: 0x1e94, 0x15c9: 0x1e98, 0x15ca: 0x1e9c, 0x15cb: 0x1ea0, - 0x15cc: 0x1ea4, 0x15cd: 0x1ea8, 0x15ce: 0x1eac, - 0x15d2: 0x1826, 0x15d3: 0x183e, 0x15d4: 0x1eb0, 0x15d5: 0x1eb4, 0x15d6: 0x1eb8, 0x15d7: 0x1ebc, - 0x15d8: 0x1ec0, 0x15d9: 0x1ec4, 0x15da: 0x1836, 0x15db: 0x1ec8, 0x15dc: 0x1ecc, 0x15dd: 0x1ed0, - 0x15de: 0x1ed4, 0x15df: 0x1846, - // Block 0x58, offset 0x1600 - 0x1600: 0x1ed8, 0x1601: 0x1ede, 0x1602: 0x1ee4, 0x1603: 0x1eea, 0x1604: 0x1ef0, 0x1605: 0x1ef6, - 0x1606: 0x1efc, 0x1607: 0x1f02, 0x1608: 0x1f08, 0x1609: 0x1f0e, 0x160a: 0x1f14, 0x160b: 0x1f1a, - 0x160c: 0x1f20, 0x160d: 0x1f26, 0x160e: 0x1f2c, 0x160f: 0x1f35, 0x1610: 0x1f3e, 0x1611: 0x1f47, - 0x1612: 0x1f50, 0x1613: 0x1f59, 0x1614: 0x1f62, 0x1615: 0x1f6b, 0x1616: 0x1f74, 0x1617: 0x1f7d, - 0x1618: 0x1f86, 0x1619: 0x1f8f, 0x161a: 0x1f98, 0x161b: 0x1fa1, 0x161c: 0x1faa, 0x161d: 0x1fb3, - 0x161e: 0x1fc5, 0x1620: 0x1fd4, 0x1621: 0x1fda, 0x1622: 0x1fe0, 0x1623: 0x1fe6, - 0x1624: 0x1fec, 0x1625: 0x1ff2, 0x1626: 0x1ff8, 0x1627: 0x1ffe, 0x1628: 0x2004, 0x1629: 0x200a, - 0x162a: 0x2010, 0x162b: 0x2016, 0x162c: 0x201c, 0x162d: 0x2022, 0x162e: 0x2028, 0x162f: 0x202e, - 0x1630: 0x2034, 0x1631: 0x203a, 0x1632: 0x2040, 0x1633: 0x2046, 0x1634: 0x204c, 0x1635: 0x2052, - 0x1636: 0x2058, 0x1637: 0x205e, 0x1638: 0x2064, 0x1639: 0x206a, 0x163a: 0x2070, 0x163b: 0x2076, - 0x163c: 0x207c, 0x163d: 0x2082, 0x163e: 0x2088, 0x163f: 0x208e, - // Block 0x59, offset 0x1640 - 0x1640: 0x2094, 0x1641: 0x209a, 0x1642: 0x20a0, 0x1643: 0x20a6, 0x1644: 0x20ac, 0x1645: 0x20b0, - 0x1646: 0x192e, 0x1647: 0x20b4, - 0x1650: 0x20b8, 0x1651: 0x20bc, - 0x1652: 0x20bf, 0x1653: 0x20c2, 0x1654: 0x20c5, 0x1655: 0x20c8, 0x1656: 0x20cb, 0x1657: 0x20ce, - 0x1658: 0x20d1, 0x1659: 0x20d4, 0x165a: 0x20d7, 0x165b: 0x20da, 0x165c: 0x20dd, 0x165d: 0x20e0, - 0x165e: 0x20e3, 0x165f: 0x20e6, 0x1660: 0x1d38, 0x1661: 0x1d44, 0x1662: 0x1d50, 0x1663: 0x1d58, - 0x1664: 0x1d78, 0x1665: 0x1d7c, 0x1666: 0x1d88, 0x1667: 0x1d90, 0x1668: 0x1d94, 0x1669: 0x1d9c, - 0x166a: 0x1da0, 0x166b: 0x1da4, 0x166c: 0x1da8, 0x166d: 0x1dac, 0x166e: 0x20e9, 0x166f: 0x20f0, - 0x1670: 0x20f7, 0x1671: 0x20fe, 0x1672: 0x2105, 0x1673: 0x210c, 0x1674: 0x2113, 0x1675: 0x211a, - 0x1676: 0x2121, 0x1677: 0x2128, 0x1678: 0x212f, 0x1679: 0x2136, 0x167a: 0x213d, 0x167b: 0x2144, - 0x167c: 0x214b, 0x167d: 0x215b, 0x167e: 0x2168, - // Block 0x5a, offset 0x1680 - 0x1680: 0x1826, 0x1681: 0x183e, 0x1682: 0x1eb0, 0x1683: 0x1eb4, 0x1684: 0x216f, 0x1685: 0x2173, - 0x1686: 0x2177, 0x1687: 0x1852, 0x1688: 0x217b, 0x1689: 0x1882, 0x168a: 0x194a, 0x168b: 0x197a, - 0x168c: 0x1976, 0x168d: 0x194e, 0x168e: 0x1abe, 0x168f: 0x18a2, 0x1690: 0x1942, 0x1691: 0x217f, - 0x1692: 0x2183, 0x1693: 0x2187, 0x1694: 0x218b, 0x1695: 0x218f, 0x1696: 0x2193, 0x1697: 0x2197, - 0x1698: 0x219b, 0x1699: 0x219f, 0x169a: 0x21a3, 0x169b: 0x18ba, 0x169c: 0x21a7, 0x169d: 0x21ab, - 0x169e: 0x21af, 0x169f: 0x21b3, 0x16a0: 0x21b7, 0x16a1: 0x21bb, 0x16a2: 0x21bf, 0x16a3: 0x21c3, - 0x16a4: 0x1eb8, 0x16a5: 0x1ebc, 0x16a6: 0x1ec0, 0x16a7: 0x21c7, 0x16a8: 0x21cb, 0x16a9: 0x21cf, - 0x16aa: 0x21d3, 0x16ab: 0x21d7, 0x16ac: 0x21db, 0x16ad: 0x21df, 0x16ae: 0x21e3, 0x16af: 0x21e7, - 0x16b0: 0x21eb, 0x16b1: 0x21ef, 0x16b2: 0x21f2, 0x16b3: 0x21f5, 0x16b4: 0x21f8, 0x16b5: 0x21fb, - 0x16b6: 0x21fe, 0x16b7: 0x2201, 0x16b8: 0x2204, 0x16b9: 0x2207, 0x16ba: 0x220a, 0x16bb: 0x220d, - 0x16bc: 0x2210, 0x16bd: 0x2213, 0x16be: 0x2216, 0x16bf: 0x2219, - // Block 0x5b, offset 0x16c0 - 0x16c0: 0x221c, 0x16c1: 0x2221, 0x16c2: 0x2226, 0x16c3: 0x222b, 0x16c4: 0x2230, 0x16c5: 0x2235, - 0x16c6: 0x223a, 0x16c7: 0x223f, 0x16c8: 0x2244, 0x16c9: 0x2249, 0x16ca: 0x224f, 0x16cb: 0x2255, - 0x16cc: 0x225b, 0x16cd: 0x225e, 0x16ce: 0x2262, 0x16cf: 0x2265, 0x16d0: 0x2269, 0x16d1: 0x226d, - 0x16d2: 0x2271, 0x16d3: 0x2275, 0x16d4: 0x2279, 0x16d5: 0x227d, 0x16d6: 0x2281, 0x16d7: 0x2285, - 0x16d8: 0x2289, 0x16d9: 0x228d, 0x16da: 0x2291, 0x16db: 0x2295, 0x16dc: 0x2299, 0x16dd: 0x229d, - 0x16de: 0x22a1, 0x16df: 0x22a5, 0x16e0: 0x22a9, 0x16e1: 0x22ad, 0x16e2: 0x22b1, 0x16e3: 0x22b5, - 0x16e4: 0x22b9, 0x16e5: 0x22bd, 0x16e6: 0x22c1, 0x16e7: 0x22c5, 0x16e8: 0x22c9, 0x16e9: 0x22cd, - 0x16ea: 0x22d1, 0x16eb: 0x22d5, 0x16ec: 0x22d9, 0x16ed: 0x22dd, 0x16ee: 0x22e1, 0x16ef: 0x22e5, - 0x16f0: 0x22e9, 0x16f1: 0x22ed, 0x16f2: 0x22f1, 0x16f3: 0x22f5, 0x16f4: 0x22f9, 0x16f5: 0x22fd, - 0x16f6: 0x2301, 0x16f7: 0x2305, 0x16f8: 0x2309, 0x16f9: 0x230d, 0x16fa: 0x2311, 0x16fb: 0x2315, - 0x16fc: 0x2319, 0x16fd: 0x231d, 0x16fe: 0x2321, - // Block 0x5c, offset 0x1700 - 0x1700: 0x2325, 0x1701: 0x2335, 0x1702: 0x2342, 0x1703: 0x2352, 0x1704: 0x235c, 0x1705: 0x236c, - 0x1706: 0x2376, 0x1707: 0x2380, 0x1708: 0x2393, 0x1709: 0x23a0, 0x170a: 0x23aa, 0x170b: 0x23b4, - 0x170c: 0x23be, 0x170d: 0x23cb, 0x170e: 0x23d8, 0x170f: 0x23e5, 0x1710: 0x23f2, 0x1711: 0x23ff, - 0x1712: 0x240c, 0x1713: 0x2419, 0x1714: 0x242c, 0x1715: 0x2433, 0x1716: 0x2446, 0x1717: 0x2459, - 0x1718: 0x2469, 0x1719: 0x2476, 0x171a: 0x2489, 0x171b: 0x249c, 0x171c: 0x24a9, 0x171d: 0x24b3, - 0x171e: 0x24bd, 0x171f: 0x24ca, 0x1720: 0x24d7, 0x1721: 0x24e7, 0x1722: 0x24f7, 0x1723: 0x2501, - 0x1724: 0x250b, 0x1725: 0x2518, 0x1726: 0x2522, 0x1727: 0x252c, 0x1728: 0x2533, 0x1729: 0x253a, - 0x172a: 0x2544, 0x172b: 0x254e, 0x172c: 0x2561, 0x172d: 0x256e, 0x172e: 0x257e, 0x172f: 0x2591, - 0x1730: 0x259e, 0x1731: 0x25a8, 0x1732: 0x25b2, 0x1733: 0x25c5, 0x1734: 0x25d2, 0x1735: 0x25e5, - 0x1736: 0x25ef, 0x1737: 0x25ff, 0x1738: 0x2609, 0x1739: 0x2616, 0x173a: 0x2620, 0x173b: 0x262d, - 0x173c: 0x263d, 0x173d: 0x264a, 0x173e: 0x265a, 0x173f: 0x2667, - // Block 0x5d, offset 0x1740 - 0x1740: 0x266e, 0x1741: 0x267e, 0x1742: 0x2688, 0x1743: 0x2692, 0x1744: 0x269f, 0x1745: 0x26a9, - 0x1746: 0x26b3, 0x1747: 0x26bd, 0x1748: 0x26cd, 0x1749: 0x26da, 0x174a: 0x26e1, 0x174b: 0x26f4, - 0x174c: 0x26fe, 0x174d: 0x270e, 0x174e: 0x271b, 0x174f: 0x2728, 0x1750: 0x2732, 0x1751: 0x273c, - 0x1752: 0x2749, 0x1753: 0x2750, 0x1754: 0x275d, 0x1755: 0x276d, 0x1756: 0x2774, 0x1757: 0x2787, - 0x1758: 0x2791, 0x1759: 0x2796, 0x175a: 0x279b, 0x175b: 0x27a0, 0x175c: 0x27a5, 0x175d: 0x27aa, - 0x175e: 0x27af, 0x175f: 0x27b4, 0x1760: 0x27b9, 0x1761: 0x27be, 0x1762: 0x27c3, 0x1763: 0x27c9, - 0x1764: 0x27cf, 0x1765: 0x27d5, 0x1766: 0x27db, 0x1767: 0x27e1, 0x1768: 0x27e7, 0x1769: 0x27ed, - 0x176a: 0x27f3, 0x176b: 0x27f9, 0x176c: 0x27ff, 0x176d: 0x2805, 0x176e: 0x280b, 0x176f: 0x2811, - 0x1770: 0x2817, 0x1771: 0x281d, 0x1772: 0x2821, 0x1773: 0x2824, 0x1774: 0x2827, 0x1775: 0x282b, - 0x1776: 0x282e, 0x1777: 0x2831, 0x1778: 0x2834, 0x1779: 0x2838, 0x177a: 0x283c, 0x177b: 0x283f, - 0x177c: 0x2846, 0x177d: 0x284d, 0x177e: 0x2854, 0x177f: 0x285b, - // Block 0x5e, offset 0x1780 - 0x1780: 0x2868, 0x1781: 0x286b, 0x1782: 0x286e, 0x1783: 0x2872, 0x1784: 0x2875, 0x1785: 0x2878, - 0x1786: 0x287b, 0x1787: 0x287e, 0x1788: 0x2881, 0x1789: 0x2885, 0x178a: 0x288a, 0x178b: 0x288d, - 0x178c: 0x2890, 0x178d: 0x2894, 0x178e: 0x2898, 0x178f: 0x289b, 0x1790: 0x289e, 0x1791: 0x28a1, - 0x1792: 0x28a5, 0x1793: 0x28a9, 0x1794: 0x28ad, 0x1795: 0x28b1, 0x1796: 0x28b5, 0x1797: 0x28b8, - 0x1798: 0x28bb, 0x1799: 0x28be, 0x179a: 0x28c1, 0x179b: 0x28c4, 0x179c: 0x28c8, 0x179d: 0x28cb, - 0x179e: 0x28ce, 0x179f: 0x28d1, 0x17a0: 0x28d5, 0x17a1: 0x28d9, 0x17a2: 0x28dc, 0x17a3: 0x28e0, - 0x17a4: 0x28e4, 0x17a5: 0x28e8, 0x17a6: 0x28eb, 0x17a7: 0x28ef, 0x17a8: 0x28f5, 0x17a9: 0x28fc, - 0x17aa: 0x28ff, 0x17ab: 0x2903, 0x17ac: 0x2907, 0x17ad: 0x290b, 0x17ae: 0x290f, 0x17af: 0x2917, - 0x17b0: 0x2920, 0x17b1: 0x2923, 0x17b2: 0x2926, 0x17b3: 0x292a, 0x17b4: 0x292d, 0x17b5: 0x2930, - 0x17b6: 0x2933, 0x17b7: 0x2937, 0x17b8: 0x293a, 0x17b9: 0x293d, 0x17ba: 0x2940, 0x17bb: 0x2943, - 0x17bc: 0x2946, 0x17bd: 0x294a, 0x17be: 0x294d, 0x17bf: 0x2950, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x2953, 0x17c1: 0x2957, 0x17c2: 0x295b, 0x17c3: 0x2960, 0x17c4: 0x2963, 0x17c5: 0x2966, - 0x17c6: 0x2969, 0x17c7: 0x2970, 0x17c8: 0x2974, 0x17c9: 0x2977, 0x17ca: 0x297a, 0x17cb: 0x297d, - 0x17cc: 0x2980, 0x17cd: 0x2983, 0x17ce: 0x2986, 0x17cf: 0x2989, 0x17d0: 0x298c, 0x17d1: 0x298f, - 0x17d2: 0x2992, 0x17d3: 0x2996, 0x17d4: 0x2999, 0x17d5: 0x299c, 0x17d6: 0x29a0, 0x17d7: 0x29a4, - 0x17d8: 0x29a7, 0x17d9: 0x29ac, 0x17da: 0x29b0, 0x17db: 0x29b3, 0x17dc: 0x29b6, 0x17dd: 0x29b9, - 0x17de: 0x29bc, 0x17df: 0x29c2, 0x17e0: 0x29c8, 0x17e1: 0x29cd, 0x17e2: 0x29d2, 0x17e3: 0x29d7, - 0x17e4: 0x29dc, 0x17e5: 0x29e1, 0x17e6: 0x29e6, 0x17e7: 0x29eb, 0x17e8: 0x29f0, 0x17e9: 0x29f5, - 0x17ea: 0x29fb, 0x17eb: 0x2a01, 0x17ec: 0x2a07, 0x17ed: 0x2a0d, 0x17ee: 0x2a13, 0x17ef: 0x2a19, - 0x17f0: 0x2a1f, 0x17f1: 0x2a25, 0x17f2: 0x2a2b, 0x17f3: 0x2a31, 0x17f4: 0x2a37, 0x17f5: 0x2a3d, - 0x17f6: 0x2a43, 0x17f7: 0x2a49, 0x17f8: 0x2a4f, 0x17f9: 0x2a55, 0x17fa: 0x2a5b, 0x17fb: 0x2a61, - 0x17fc: 0x2a67, 0x17fd: 0x2a6d, 0x17fe: 0x2a73, 0x17ff: 0x2a79, - // Block 0x60, offset 0x1800 - 0x1830: 0x2a7d, - // Block 0x61, offset 0x1840 - 0x1840: 0x2a81, 0x1841: 0x2a85, 0x1842: 0x1a9e, 0x1843: 0x2a89, 0x1844: 0x2a8d, 0x1845: 0x2a91, - 0x1846: 0x2a95, 0x1847: 0x1b76, 0x1848: 0x1b76, 0x1849: 0x2a99, 0x184a: 0x1abe, 0x184b: 0x2a9d, - 0x184c: 0x2aa1, 0x184d: 0x2aa5, 0x184e: 0x2aa9, 0x184f: 0x2aad, 0x1850: 0x2ab1, 0x1851: 0x2ab5, - 0x1852: 0x2ab9, 0x1853: 0x2abd, 0x1854: 0x2ac1, 0x1855: 0x2ac5, 0x1856: 0x2ac9, 0x1857: 0x2acd, - 0x1858: 0x2ad1, 0x1859: 0x2ad5, 0x185a: 0x2ad9, 0x185b: 0x2add, 0x185c: 0x2ae1, 0x185d: 0x2ae5, - 0x185e: 0x2ae9, 0x185f: 0x2aed, 0x1860: 0x2af1, 0x1861: 0x2af5, 0x1862: 0x2af9, 0x1863: 0x2afd, - 0x1864: 0x2b01, 0x1865: 0x2b05, 0x1866: 0x2b09, 0x1867: 0x2b0d, 0x1868: 0x2b11, 0x1869: 0x2b15, - 0x186a: 0x2b19, 0x186b: 0x2b1d, 0x186c: 0x2b21, 0x186d: 0x2b25, 0x186e: 0x2b29, 0x186f: 0x2b2d, - 0x1870: 0x2b31, 0x1871: 0x2b35, 0x1872: 0x2b39, 0x1873: 0x2b3d, 0x1874: 0x1a16, 0x1875: 0x2b41, - 0x1876: 0x2b45, 0x1877: 0x2b49, 0x1878: 0x2b4d, 0x1879: 0x2b51, 0x187a: 0x2b55, 0x187b: 0x2b59, - 0x187c: 0x2b5d, 0x187d: 0x2b61, 0x187e: 0x2b65, 0x187f: 0x2b69, - // Block 0x62, offset 0x1880 - 0x1880: 0x1b3a, 0x1881: 0x2b6d, 0x1882: 0x2b71, 0x1883: 0x2b75, 0x1884: 0x2b79, 0x1885: 0x2b7d, - 0x1886: 0x2b81, 0x1887: 0x2b85, 0x1888: 0x2b89, 0x1889: 0x2b8d, 0x188a: 0x2b91, 0x188b: 0x2b95, - 0x188c: 0x2b99, 0x188d: 0x2b9d, 0x188e: 0x2ba1, 0x188f: 0x2ba5, 0x1890: 0x2ba9, 0x1891: 0x2bad, - 0x1892: 0x2bb1, 0x1893: 0x2bb5, 0x1894: 0x2bb9, 0x1895: 0x2bbd, 0x1896: 0x2bc1, 0x1897: 0x2bc5, - 0x1898: 0x2bc9, 0x1899: 0x2bcd, 0x189a: 0x2bd1, 0x189b: 0x2bd5, 0x189c: 0x2ac1, 0x189d: 0x2bd9, - 0x189e: 0x2bdd, 0x189f: 0x2be1, 0x18a0: 0x2be5, 0x18a1: 0x2be9, 0x18a2: 0x2bed, 0x18a3: 0x2bf1, - 0x18a4: 0x2bf5, 0x18a5: 0x2bf9, 0x18a6: 0x2bfd, 0x18a7: 0x2c01, 0x18a8: 0x2c05, 0x18a9: 0x2c09, - 0x18aa: 0x2c0d, 0x18ab: 0x2c11, 0x18ac: 0x2c15, 0x18ad: 0x2c19, 0x18ae: 0x2c1d, 0x18af: 0x2c21, - 0x18b0: 0x2c25, 0x18b1: 0x1aa6, 0x18b2: 0x2c29, 0x18b3: 0x2c2d, 0x18b4: 0x2c31, 0x18b5: 0x2c35, - 0x18b6: 0x2c39, 0x18b7: 0x2c3d, 0x18b8: 0x2c41, 0x18b9: 0x2c45, 0x18ba: 0x2c49, 0x18bb: 0x2c4d, - 0x18bc: 0x2c51, 0x18bd: 0x2c55, 0x18be: 0x2c59, 0x18bf: 0x2c5d, - // Block 0x63, offset 0x18c0 - 0x18c0: 0x2c61, 0x18c1: 0x18ba, 0x18c2: 0x2c65, 0x18c3: 0x2c69, 0x18c4: 0x2c6d, 0x18c5: 0x2c71, - 0x18c6: 0x2c75, 0x18c7: 0x2c79, 0x18c8: 0x2c7d, 0x18c9: 0x2c81, 0x18ca: 0x186e, 0x18cb: 0x2c85, - 0x18cc: 0x2c89, 0x18cd: 0x2c8d, 0x18ce: 0x2c91, 0x18cf: 0x2c95, 0x18d0: 0x2c99, 0x18d1: 0x2c9d, - 0x18d2: 0x2ca1, 0x18d3: 0x2ca5, 0x18d4: 0x2ca9, 0x18d5: 0x2cad, 0x18d6: 0x2cb1, 0x18d7: 0x2cb5, - 0x18d8: 0x2cb9, 0x18d9: 0x2cbd, 0x18da: 0x2cc1, 0x18db: 0x2cc5, 0x18dc: 0x2cc9, 0x18dd: 0x2ccd, - 0x18de: 0x2cd1, 0x18df: 0x2cd5, 0x18e0: 0x2cd9, 0x18e1: 0x2c21, 0x18e2: 0x2cdd, 0x18e3: 0x2ce1, - 0x18e4: 0x2ce5, 0x18e5: 0x2ce9, 0x18e6: 0x2ced, 0x18e7: 0x2cf1, 0x18e8: 0x2cf5, 0x18e9: 0x2cf9, - 0x18ea: 0x2be1, 0x18eb: 0x2cfd, 0x18ec: 0x2d01, 0x18ed: 0x2d05, 0x18ee: 0x2d09, 0x18ef: 0x2d0d, - 0x18f0: 0x2d11, 0x18f1: 0x2d15, 0x18f2: 0x2d19, 0x18f3: 0x2d1d, 0x18f4: 0x2d21, 0x18f5: 0x2d25, - 0x18f6: 0x2d29, 0x18f7: 0x2d2d, 0x18f8: 0x2d31, 0x18f9: 0x2d35, 0x18fa: 0x2d39, 0x18fb: 0x2d3d, - 0x18fc: 0x2d41, 0x18fd: 0x2d45, 0x18fe: 0x2d49, 0x18ff: 0x2ac1, - // Block 0x64, offset 0x1900 - 0x1900: 0x2d4d, 0x1901: 0x2d51, 0x1902: 0x2d55, 0x1903: 0x2d59, 0x1904: 0x1b72, 0x1905: 0x2d5d, - 0x1906: 0x2d61, 0x1907: 0x2d65, 0x1908: 0x2d69, 0x1909: 0x2d6d, 0x190a: 0x2d71, 0x190b: 0x2d75, - 0x190c: 0x2d79, 0x190d: 0x2d7d, 0x190e: 0x2d81, 0x190f: 0x2d85, 0x1910: 0x2d89, 0x1911: 0x2173, - 0x1912: 0x2d8d, 0x1913: 0x2d91, 0x1914: 0x2d95, 0x1915: 0x2d99, 0x1916: 0x2d9d, 0x1917: 0x2da1, - 0x1918: 0x2da5, 0x1919: 0x2da9, 0x191a: 0x2dad, 0x191b: 0x2be9, 0x191c: 0x2db1, 0x191d: 0x2db5, - 0x191e: 0x2db9, 0x191f: 0x2dbd, 0x1920: 0x2dc1, 0x1921: 0x2dc5, 0x1922: 0x2dc9, 0x1923: 0x2dcd, - 0x1924: 0x2dd1, 0x1925: 0x2dd5, 0x1926: 0x2dd9, 0x1927: 0x2ddd, 0x1928: 0x2de1, 0x1929: 0x1aba, - 0x192a: 0x2de5, 0x192b: 0x2de9, 0x192c: 0x2ded, 0x192d: 0x2df1, 0x192e: 0x2df5, 0x192f: 0x2df9, - 0x1930: 0x2dfd, 0x1931: 0x2e01, 0x1932: 0x2e05, 0x1933: 0x2e09, 0x1934: 0x2e0d, 0x1935: 0x2e11, - 0x1936: 0x2e15, 0x1937: 0x19f6, 0x1938: 0x2e19, 0x1939: 0x2e1d, 0x193a: 0x2e21, 0x193b: 0x2e25, - 0x193c: 0x2e29, 0x193d: 0x2e2d, 0x193e: 0x2e31, 0x193f: 0x2e35, - // Block 0x65, offset 0x1940 - 0x1940: 0x2e39, 0x1941: 0x2e3d, 0x1942: 0x2e41, 0x1943: 0x2e45, 0x1944: 0x2e49, 0x1945: 0x2e4d, - 0x1946: 0x2e51, 0x1947: 0x2e55, 0x1948: 0x1a62, 0x1949: 0x2e59, 0x194a: 0x1a6e, 0x194b: 0x2e5d, - 0x194c: 0x2e61, 0x194d: 0x2e65, 0x1950: 0x2e69, - 0x1952: 0x2e6d, 0x1955: 0x2e71, 0x1956: 0x2e75, 0x1957: 0x2e79, - 0x1958: 0x2e7d, 0x1959: 0x2e81, 0x195a: 0x2e85, 0x195b: 0x2e89, 0x195c: 0x2e8d, 0x195d: 0x2e91, - 0x195e: 0x1a12, 0x1960: 0x2e95, 0x1962: 0x2e99, - 0x1965: 0x2e9d, 0x1966: 0x2ea1, - 0x196a: 0x2ea5, 0x196b: 0x2ea9, 0x196c: 0x2ead, 0x196d: 0x2eb1, - 0x1970: 0x2eb5, 0x1971: 0x2eb9, 0x1972: 0x2ebd, 0x1973: 0x2ec1, 0x1974: 0x2ec5, 0x1975: 0x2ec9, - 0x1976: 0x2ecd, 0x1977: 0x2ed1, 0x1978: 0x2ed5, 0x1979: 0x2ed9, 0x197a: 0x2edd, 0x197b: 0x2ee1, - 0x197c: 0x18d6, 0x197d: 0x2ee5, 0x197e: 0x2ee9, 0x197f: 0x2eed, - // Block 0x66, offset 0x1980 - 0x1980: 0x2ef1, 0x1981: 0x2ef5, 0x1982: 0x2ef9, 0x1983: 0x2efd, 0x1984: 0x2f01, 0x1985: 0x2f05, - 0x1986: 0x2f09, 0x1987: 0x2f0d, 0x1988: 0x2f11, 0x1989: 0x2f15, 0x198a: 0x2f19, 0x198b: 0x2f1d, - 0x198c: 0x2187, 0x198d: 0x2f21, 0x198e: 0x2f25, 0x198f: 0x2f29, 0x1990: 0x2f2d, 0x1991: 0x2197, - 0x1992: 0x2f31, 0x1993: 0x2f35, 0x1994: 0x2f39, 0x1995: 0x2f3d, 0x1996: 0x2f41, 0x1997: 0x2cb1, - 0x1998: 0x2f45, 0x1999: 0x2f49, 0x199a: 0x2f4d, 0x199b: 0x2f51, 0x199c: 0x2f55, 0x199d: 0x2f59, - 0x199e: 0x2f59, 0x199f: 0x2f5d, 0x19a0: 0x2f61, 0x19a1: 0x2f65, 0x19a2: 0x2f69, 0x19a3: 0x2f6d, - 0x19a4: 0x2f71, 0x19a5: 0x2f75, 0x19a6: 0x2f79, 0x19a7: 0x2e9d, 0x19a8: 0x2f7d, 0x19a9: 0x2f81, - 0x19aa: 0x2f85, 0x19ab: 0x2f89, 0x19ac: 0x2f8d, 0x19ad: 0x2f92, - 0x19b0: 0x2f96, 0x19b1: 0x2f9a, 0x19b2: 0x2f9e, 0x19b3: 0x2fa2, 0x19b4: 0x2fa6, 0x19b5: 0x2faa, - 0x19b6: 0x2fae, 0x19b7: 0x2fb2, 0x19b8: 0x2ecd, 0x19b9: 0x2fb6, 0x19ba: 0x2fba, 0x19bb: 0x2fbe, - 0x19bc: 0x2e69, 0x19bd: 0x2fc2, 0x19be: 0x2fc6, 0x19bf: 0x2fca, - // Block 0x67, offset 0x19c0 - 0x19c0: 0x2fce, 0x19c1: 0x2fd2, 0x19c2: 0x2fd6, 0x19c3: 0x2fda, 0x19c4: 0x2fde, 0x19c5: 0x2fe2, - 0x19c6: 0x2fe6, 0x19c7: 0x2fea, 0x19c8: 0x2fee, 0x19c9: 0x2eed, 0x19ca: 0x2ff2, 0x19cb: 0x2ef1, - 0x19cc: 0x2ff6, 0x19cd: 0x2ffa, 0x19ce: 0x2ffe, 0x19cf: 0x3002, 0x19d0: 0x3006, 0x19d1: 0x2e6d, - 0x19d2: 0x2b15, 0x19d3: 0x300a, 0x19d4: 0x300e, 0x19d5: 0x195a, 0x19d6: 0x2c25, 0x19d7: 0x2d71, - 0x19d8: 0x3012, 0x19d9: 0x3016, 0x19da: 0x2f0d, 0x19db: 0x301a, 0x19dc: 0x2f11, 0x19dd: 0x301e, - 0x19de: 0x3022, 0x19df: 0x3026, 0x19e0: 0x2e75, 0x19e1: 0x302a, 0x19e2: 0x302e, 0x19e3: 0x3032, - 0x19e4: 0x3036, 0x19e5: 0x303a, 0x19e6: 0x2e79, 0x19e7: 0x303e, 0x19e8: 0x3042, 0x19e9: 0x3046, - 0x19ea: 0x304a, 0x19eb: 0x304e, 0x19ec: 0x3052, 0x19ed: 0x2f41, 0x19ee: 0x3056, 0x19ef: 0x305a, - 0x19f0: 0x2cb1, 0x19f1: 0x305e, 0x19f2: 0x2f51, 0x19f3: 0x3062, 0x19f4: 0x3066, 0x19f5: 0x306a, - 0x19f6: 0x306e, 0x19f7: 0x3072, 0x19f8: 0x2f65, 0x19f9: 0x3076, 0x19fa: 0x2e99, 0x19fb: 0x307a, - 0x19fc: 0x2f69, 0x19fd: 0x2bd9, 0x19fe: 0x307e, 0x19ff: 0x2f6d, - // Block 0x68, offset 0x1a00 - 0x1a00: 0x3082, 0x1a01: 0x2f75, 0x1a02: 0x3086, 0x1a03: 0x308a, 0x1a04: 0x308e, 0x1a05: 0x3092, - 0x1a06: 0x3096, 0x1a07: 0x2f7d, 0x1a08: 0x2e8d, 0x1a09: 0x309a, 0x1a0a: 0x2f81, 0x1a0b: 0x309e, - 0x1a0c: 0x2f85, 0x1a0d: 0x30a2, 0x1a0e: 0x1b76, 0x1a0f: 0x30a6, 0x1a10: 0x30ab, 0x1a11: 0x30b0, - 0x1a12: 0x30b5, 0x1a13: 0x30b9, 0x1a14: 0x30bd, 0x1a15: 0x30c1, 0x1a16: 0x30c6, 0x1a17: 0x30cb, - 0x1a18: 0x30d0, 0x1a19: 0x30d4, - // Block 0x69, offset 0x1a40 - 0x1a40: 0x30d8, 0x1a41: 0x30db, 0x1a42: 0x30de, 0x1a43: 0x30e1, 0x1a44: 0x30e5, 0x1a45: 0x30e9, - 0x1a46: 0x30e9, - 0x1a53: 0x30ec, 0x1a54: 0x30f1, 0x1a55: 0x30f6, 0x1a56: 0x30fb, 0x1a57: 0x3100, - 0x1a5d: 0x3105, - 0x1a5f: 0x310a, 0x1a60: 0x310f, 0x1a61: 0x14db, 0x1a62: 0x14e4, 0x1a63: 0x3112, - 0x1a64: 0x3115, 0x1a65: 0x3118, 0x1a66: 0x311b, 0x1a67: 0x311e, 0x1a68: 0x3121, 0x1a69: 0x1494, - 0x1a6a: 0x3124, 0x1a6b: 0x3129, 0x1a6c: 0x312e, 0x1a6d: 0x3135, 0x1a6e: 0x313c, 0x1a6f: 0x3141, - 0x1a70: 0x3146, 0x1a71: 0x314b, 0x1a72: 0x3150, 0x1a73: 0x3155, 0x1a74: 0x315a, 0x1a75: 0x315f, - 0x1a76: 0x3164, 0x1a78: 0x3169, 0x1a79: 0x316e, 0x1a7a: 0x3173, 0x1a7b: 0x3178, - 0x1a7c: 0x317d, 0x1a7e: 0x3182, - // Block 0x6a, offset 0x1a80 - 0x1a80: 0x3187, 0x1a81: 0x318c, 0x1a83: 0x3191, 0x1a84: 0x3196, - 0x1a86: 0x319b, 0x1a87: 0x31a0, 0x1a88: 0x31a5, 0x1a89: 0x31aa, 0x1a8a: 0x31af, 0x1a8b: 0x31b4, - 0x1a8c: 0x31b9, 0x1a8d: 0x31be, 0x1a8e: 0x31c3, 0x1a8f: 0x31c8, 0x1a90: 0x31cd, 0x1a91: 0x31cd, - 0x1a92: 0x31d0, 0x1a93: 0x31d0, 0x1a94: 0x31d0, 0x1a95: 0x31d0, 0x1a96: 0x31d3, 0x1a97: 0x31d3, - 0x1a98: 0x31d3, 0x1a99: 0x31d3, 0x1a9a: 0x31d6, 0x1a9b: 0x31d6, 0x1a9c: 0x31d6, 0x1a9d: 0x31d6, - 0x1a9e: 0x31d9, 0x1a9f: 0x31d9, 0x1aa0: 0x31d9, 0x1aa1: 0x31d9, 0x1aa2: 0x31dc, 0x1aa3: 0x31dc, - 0x1aa4: 0x31dc, 0x1aa5: 0x31dc, 0x1aa6: 0x31df, 0x1aa7: 0x31df, 0x1aa8: 0x31df, 0x1aa9: 0x31df, - 0x1aaa: 0x31e2, 0x1aab: 0x31e2, 0x1aac: 0x31e2, 0x1aad: 0x31e2, 0x1aae: 0x31e5, 0x1aaf: 0x31e5, - 0x1ab0: 0x31e5, 0x1ab1: 0x31e5, 0x1ab2: 0x31e8, 0x1ab3: 0x31e8, 0x1ab4: 0x31e8, 0x1ab5: 0x31e8, - 0x1ab6: 0x31eb, 0x1ab7: 0x31eb, 0x1ab8: 0x31eb, 0x1ab9: 0x31eb, 0x1aba: 0x31ee, 0x1abb: 0x31ee, - 0x1abc: 0x31ee, 0x1abd: 0x31ee, 0x1abe: 0x31f1, 0x1abf: 0x31f1, - // Block 0x6b, offset 0x1ac0 - 0x1ac0: 0x31f1, 0x1ac1: 0x31f1, 0x1ac2: 0x31f4, 0x1ac3: 0x31f4, 0x1ac4: 0x31f7, 0x1ac5: 0x31f7, - 0x1ac6: 0x31fa, 0x1ac7: 0x31fa, 0x1ac8: 0x31fd, 0x1ac9: 0x31fd, 0x1aca: 0x3200, 0x1acb: 0x3200, - 0x1acc: 0x3203, 0x1acd: 0x3203, 0x1ace: 0x3206, 0x1acf: 0x3206, 0x1ad0: 0x3206, 0x1ad1: 0x3206, - 0x1ad2: 0x3209, 0x1ad3: 0x3209, 0x1ad4: 0x3209, 0x1ad5: 0x3209, 0x1ad6: 0x320c, 0x1ad7: 0x320c, - 0x1ad8: 0x320c, 0x1ad9: 0x320c, 0x1ada: 0x320f, 0x1adb: 0x320f, 0x1adc: 0x320f, 0x1add: 0x320f, - 0x1ade: 0x3212, 0x1adf: 0x3212, 0x1ae0: 0x3215, 0x1ae1: 0x3215, 0x1ae2: 0x3215, 0x1ae3: 0x3215, - 0x1ae4: 0x06ba, 0x1ae5: 0x06ba, 0x1ae6: 0x3218, 0x1ae7: 0x3218, 0x1ae8: 0x3218, 0x1ae9: 0x3218, - 0x1aea: 0x321b, 0x1aeb: 0x321b, 0x1aec: 0x321b, 0x1aed: 0x321b, 0x1aee: 0x321e, 0x1aef: 0x321e, - 0x1af0: 0x06c4, 0x1af1: 0x06c4, - // Block 0x6c, offset 0x1b00 - 0x1b13: 0x3221, 0x1b14: 0x3221, 0x1b15: 0x3221, 0x1b16: 0x3221, 0x1b17: 0x3224, - 0x1b18: 0x3224, 0x1b19: 0x3227, 0x1b1a: 0x3227, 0x1b1b: 0x322a, 0x1b1c: 0x322a, 0x1b1d: 0x06b0, - 0x1b1e: 0x322d, 0x1b1f: 0x322d, 0x1b20: 0x3230, 0x1b21: 0x3230, 0x1b22: 0x3233, 0x1b23: 0x3233, - 0x1b24: 0x3236, 0x1b25: 0x3236, 0x1b26: 0x3236, 0x1b27: 0x3236, 0x1b28: 0x3239, 0x1b29: 0x3239, - 0x1b2a: 0x323c, 0x1b2b: 0x323c, 0x1b2c: 0x3243, 0x1b2d: 0x3243, 0x1b2e: 0x324a, 0x1b2f: 0x324a, - 0x1b30: 0x3251, 0x1b31: 0x3251, 0x1b32: 0x3258, 0x1b33: 0x3258, 0x1b34: 0x325f, 0x1b35: 0x325f, - 0x1b36: 0x3266, 0x1b37: 0x3266, 0x1b38: 0x3266, 0x1b39: 0x326d, 0x1b3a: 0x326d, 0x1b3b: 0x326d, - 0x1b3c: 0x3274, 0x1b3d: 0x3274, 0x1b3e: 0x3274, 0x1b3f: 0x3274, - // Block 0x6d, offset 0x1b40 - 0x1b40: 0x3277, 0x1b41: 0x327e, 0x1b42: 0x3285, 0x1b43: 0x326d, 0x1b44: 0x328c, 0x1b45: 0x3293, - 0x1b46: 0x3298, 0x1b47: 0x329d, 0x1b48: 0x32a2, 0x1b49: 0x32a7, 0x1b4a: 0x32ac, 0x1b4b: 0x32b1, - 0x1b4c: 0x32b6, 0x1b4d: 0x32bb, 0x1b4e: 0x32c0, 0x1b4f: 0x32c5, 0x1b50: 0x32ca, 0x1b51: 0x32cf, - 0x1b52: 0x32d4, 0x1b53: 0x32d9, 0x1b54: 0x32de, 0x1b55: 0x32e3, 0x1b56: 0x32e8, 0x1b57: 0x32ed, - 0x1b58: 0x32f2, 0x1b59: 0x32f7, 0x1b5a: 0x32fc, 0x1b5b: 0x3301, 0x1b5c: 0x3306, 0x1b5d: 0x330b, - 0x1b5e: 0x3310, 0x1b5f: 0x3315, 0x1b60: 0x331a, 0x1b61: 0x331f, 0x1b62: 0x3324, 0x1b63: 0x3329, - 0x1b64: 0x332e, 0x1b65: 0x3333, 0x1b66: 0x3338, 0x1b67: 0x333d, 0x1b68: 0x3342, 0x1b69: 0x3347, - 0x1b6a: 0x334c, 0x1b6b: 0x3351, 0x1b6c: 0x3356, 0x1b6d: 0x335b, 0x1b6e: 0x3360, 0x1b6f: 0x3365, - 0x1b70: 0x336a, 0x1b71: 0x336f, 0x1b72: 0x3374, 0x1b73: 0x3379, 0x1b74: 0x337e, 0x1b75: 0x3383, - 0x1b76: 0x3388, 0x1b77: 0x338d, 0x1b78: 0x3392, 0x1b79: 0x3397, 0x1b7a: 0x339c, 0x1b7b: 0x33a1, - 0x1b7c: 0x33a6, 0x1b7d: 0x33ab, 0x1b7e: 0x33b0, 0x1b7f: 0x33b5, - // Block 0x6e, offset 0x1b80 - 0x1b80: 0x33ba, 0x1b81: 0x33bf, 0x1b82: 0x33c4, 0x1b83: 0x33c9, 0x1b84: 0x33ce, 0x1b85: 0x33d3, - 0x1b86: 0x33d8, 0x1b87: 0x33dd, 0x1b88: 0x33e2, 0x1b89: 0x33e7, 0x1b8a: 0x33ec, 0x1b8b: 0x33f1, - 0x1b8c: 0x33f6, 0x1b8d: 0x33fb, 0x1b8e: 0x3400, 0x1b8f: 0x3405, 0x1b90: 0x340a, 0x1b91: 0x340f, - 0x1b92: 0x3414, 0x1b93: 0x3419, 0x1b94: 0x341e, 0x1b95: 0x3423, 0x1b96: 0x3428, 0x1b97: 0x342d, - 0x1b98: 0x3432, 0x1b99: 0x3437, 0x1b9a: 0x343c, 0x1b9b: 0x3441, 0x1b9c: 0x3446, 0x1b9d: 0x344b, - 0x1b9e: 0x3450, 0x1b9f: 0x3456, 0x1ba0: 0x345c, 0x1ba1: 0x3462, 0x1ba2: 0x3468, 0x1ba3: 0x346e, - 0x1ba4: 0x3474, 0x1ba5: 0x347b, 0x1ba6: 0x3285, 0x1ba7: 0x3482, 0x1ba8: 0x326d, 0x1ba9: 0x328c, - 0x1baa: 0x3489, 0x1bab: 0x348e, 0x1bac: 0x32a2, 0x1bad: 0x3493, 0x1bae: 0x32a7, 0x1baf: 0x32ac, - 0x1bb0: 0x3498, 0x1bb1: 0x349d, 0x1bb2: 0x32c0, 0x1bb3: 0x34a2, 0x1bb4: 0x32c5, 0x1bb5: 0x32ca, - 0x1bb6: 0x34a7, 0x1bb7: 0x34ac, 0x1bb8: 0x32d4, 0x1bb9: 0x34b1, 0x1bba: 0x32d9, 0x1bbb: 0x32de, - 0x1bbc: 0x336f, 0x1bbd: 0x3374, 0x1bbe: 0x3383, 0x1bbf: 0x3388, - // Block 0x6f, offset 0x1bc0 - 0x1bc0: 0x338d, 0x1bc1: 0x33a1, 0x1bc2: 0x33a6, 0x1bc3: 0x33ab, 0x1bc4: 0x33b0, 0x1bc5: 0x33c4, - 0x1bc6: 0x33c9, 0x1bc7: 0x33ce, 0x1bc8: 0x34b6, 0x1bc9: 0x33e2, 0x1bca: 0x34bb, 0x1bcb: 0x34c0, - 0x1bcc: 0x3400, 0x1bcd: 0x34c5, 0x1bce: 0x3405, 0x1bcf: 0x340a, 0x1bd0: 0x344b, 0x1bd1: 0x34ca, - 0x1bd2: 0x34cf, 0x1bd3: 0x3432, 0x1bd4: 0x34d4, 0x1bd5: 0x3437, 0x1bd6: 0x343c, 0x1bd7: 0x3277, - 0x1bd8: 0x327e, 0x1bd9: 0x34d9, 0x1bda: 0x3285, 0x1bdb: 0x34e0, 0x1bdc: 0x3293, 0x1bdd: 0x3298, - 0x1bde: 0x329d, 0x1bdf: 0x32a2, 0x1be0: 0x34e7, 0x1be1: 0x32b1, 0x1be2: 0x32b6, 0x1be3: 0x32bb, - 0x1be4: 0x32c0, 0x1be5: 0x34ec, 0x1be6: 0x32d4, 0x1be7: 0x32e3, 0x1be8: 0x32e8, 0x1be9: 0x32ed, - 0x1bea: 0x32f2, 0x1beb: 0x32f7, 0x1bec: 0x3301, 0x1bed: 0x3306, 0x1bee: 0x330b, 0x1bef: 0x3310, - 0x1bf0: 0x3315, 0x1bf1: 0x331a, 0x1bf2: 0x34f1, 0x1bf3: 0x331f, 0x1bf4: 0x3324, 0x1bf5: 0x3329, - 0x1bf6: 0x332e, 0x1bf7: 0x3333, 0x1bf8: 0x3338, 0x1bf9: 0x3342, 0x1bfa: 0x3347, 0x1bfb: 0x334c, - 0x1bfc: 0x3351, 0x1bfd: 0x3356, 0x1bfe: 0x335b, 0x1bff: 0x3360, - // Block 0x70, offset 0x1c00 - 0x1c00: 0x3365, 0x1c01: 0x336a, 0x1c02: 0x3379, 0x1c03: 0x337e, 0x1c04: 0x3392, 0x1c05: 0x3397, - 0x1c06: 0x339c, 0x1c07: 0x33a1, 0x1c08: 0x33a6, 0x1c09: 0x33b5, 0x1c0a: 0x33ba, 0x1c0b: 0x33bf, - 0x1c0c: 0x33c4, 0x1c0d: 0x34f6, 0x1c0e: 0x33d3, 0x1c0f: 0x33d8, 0x1c10: 0x33dd, 0x1c11: 0x33e2, - 0x1c12: 0x33f1, 0x1c13: 0x33f6, 0x1c14: 0x33fb, 0x1c15: 0x3400, 0x1c16: 0x34fb, 0x1c17: 0x340f, - 0x1c18: 0x3414, 0x1c19: 0x3500, 0x1c1a: 0x3423, 0x1c1b: 0x3428, 0x1c1c: 0x342d, 0x1c1d: 0x3432, - 0x1c1e: 0x3505, 0x1c1f: 0x3285, 0x1c20: 0x34e0, 0x1c21: 0x32a2, 0x1c22: 0x34e7, 0x1c23: 0x32c0, - 0x1c24: 0x34ec, 0x1c25: 0x32d4, 0x1c26: 0x350a, 0x1c27: 0x3315, 0x1c28: 0x350f, 0x1c29: 0x3514, - 0x1c2a: 0x3519, 0x1c2b: 0x33a1, 0x1c2c: 0x33a6, 0x1c2d: 0x33c4, 0x1c2e: 0x3400, 0x1c2f: 0x34fb, - 0x1c30: 0x3432, 0x1c31: 0x3505, 0x1c32: 0x351e, 0x1c33: 0x3525, 0x1c34: 0x352c, 0x1c35: 0x3533, - 0x1c36: 0x3538, 0x1c37: 0x353d, 0x1c38: 0x3542, 0x1c39: 0x3547, 0x1c3a: 0x354c, 0x1c3b: 0x3551, - 0x1c3c: 0x3556, 0x1c3d: 0x355b, 0x1c3e: 0x3560, 0x1c3f: 0x3565, - // Block 0x71, offset 0x1c40 - 0x1c40: 0x356a, 0x1c41: 0x356f, 0x1c42: 0x3574, 0x1c43: 0x3579, 0x1c44: 0x357e, 0x1c45: 0x3583, - 0x1c46: 0x3588, 0x1c47: 0x358d, 0x1c48: 0x3592, 0x1c49: 0x3597, 0x1c4a: 0x359c, 0x1c4b: 0x35a1, - 0x1c4c: 0x3514, 0x1c4d: 0x35a6, 0x1c4e: 0x35ab, 0x1c4f: 0x35b0, 0x1c50: 0x35b5, 0x1c51: 0x3533, - 0x1c52: 0x3538, 0x1c53: 0x353d, 0x1c54: 0x3542, 0x1c55: 0x3547, 0x1c56: 0x354c, 0x1c57: 0x3551, - 0x1c58: 0x3556, 0x1c59: 0x355b, 0x1c5a: 0x3560, 0x1c5b: 0x3565, 0x1c5c: 0x356a, 0x1c5d: 0x356f, - 0x1c5e: 0x3574, 0x1c5f: 0x3579, 0x1c60: 0x357e, 0x1c61: 0x3583, 0x1c62: 0x3588, 0x1c63: 0x358d, - 0x1c64: 0x3592, 0x1c65: 0x3597, 0x1c66: 0x359c, 0x1c67: 0x35a1, 0x1c68: 0x3514, 0x1c69: 0x35a6, - 0x1c6a: 0x35ab, 0x1c6b: 0x35b0, 0x1c6c: 0x35b5, 0x1c6d: 0x3597, 0x1c6e: 0x359c, 0x1c6f: 0x35a1, - 0x1c70: 0x3514, 0x1c71: 0x350f, 0x1c72: 0x3519, 0x1c73: 0x333d, 0x1c74: 0x3306, 0x1c75: 0x330b, - 0x1c76: 0x3310, 0x1c77: 0x3597, 0x1c78: 0x359c, 0x1c79: 0x35a1, 0x1c7a: 0x333d, 0x1c7b: 0x3342, - 0x1c7c: 0x35ba, 0x1c7d: 0x35ba, - // Block 0x72, offset 0x1c80 - 0x1c90: 0x35bf, 0x1c91: 0x35c6, - 0x1c92: 0x35c6, 0x1c93: 0x35cd, 0x1c94: 0x35d4, 0x1c95: 0x35db, 0x1c96: 0x35e2, 0x1c97: 0x35e9, - 0x1c98: 0x35f0, 0x1c99: 0x35f0, 0x1c9a: 0x35f7, 0x1c9b: 0x35fe, 0x1c9c: 0x3605, 0x1c9d: 0x360c, - 0x1c9e: 0x3613, 0x1c9f: 0x361a, 0x1ca0: 0x361a, 0x1ca1: 0x3621, 0x1ca2: 0x3628, 0x1ca3: 0x3628, - 0x1ca4: 0x362f, 0x1ca5: 0x362f, 0x1ca6: 0x3636, 0x1ca7: 0x363d, 0x1ca8: 0x363d, 0x1ca9: 0x3644, - 0x1caa: 0x364b, 0x1cab: 0x364b, 0x1cac: 0x3652, 0x1cad: 0x3652, 0x1cae: 0x3659, 0x1caf: 0x3660, - 0x1cb0: 0x3660, 0x1cb1: 0x3667, 0x1cb2: 0x3667, 0x1cb3: 0x366e, 0x1cb4: 0x3675, 0x1cb5: 0x367c, - 0x1cb6: 0x3683, 0x1cb7: 0x3683, 0x1cb8: 0x368a, 0x1cb9: 0x3691, 0x1cba: 0x3698, 0x1cbb: 0x369f, - 0x1cbc: 0x36a6, 0x1cbd: 0x36a6, 0x1cbe: 0x36ad, 0x1cbf: 0x36b4, - // Block 0x73, offset 0x1cc0 - 0x1cc0: 0x36bb, 0x1cc1: 0x36c2, 0x1cc2: 0x36c9, 0x1cc3: 0x36d0, 0x1cc4: 0x36d0, 0x1cc5: 0x36d7, - 0x1cc6: 0x36d7, 0x1cc7: 0x36de, 0x1cc8: 0x36de, 0x1cc9: 0x36e5, 0x1cca: 0x36ec, 0x1ccb: 0x36f3, - 0x1ccc: 0x36fa, 0x1ccd: 0x3701, 0x1cce: 0x3708, 0x1ccf: 0x370f, - 0x1cd2: 0x3716, 0x1cd3: 0x371d, 0x1cd4: 0x3724, 0x1cd5: 0x372b, 0x1cd6: 0x3732, 0x1cd7: 0x3739, - 0x1cd8: 0x3739, 0x1cd9: 0x3740, 0x1cda: 0x3747, 0x1cdb: 0x374e, 0x1cdc: 0x3755, 0x1cdd: 0x3755, - 0x1cde: 0x375c, 0x1cdf: 0x3763, 0x1ce0: 0x376a, 0x1ce1: 0x3771, 0x1ce2: 0x3778, 0x1ce3: 0x377f, - 0x1ce4: 0x3786, 0x1ce5: 0x378d, 0x1ce6: 0x3794, 0x1ce7: 0x379b, 0x1ce8: 0x37a2, 0x1ce9: 0x37a9, - 0x1cea: 0x37b0, 0x1ceb: 0x37b7, 0x1cec: 0x37be, 0x1ced: 0x37c5, 0x1cee: 0x37cc, 0x1cef: 0x37d3, - 0x1cf0: 0x37da, 0x1cf1: 0x37e1, 0x1cf2: 0x37e8, 0x1cf3: 0x37ef, 0x1cf4: 0x36ad, 0x1cf5: 0x36bb, - 0x1cf6: 0x37f6, 0x1cf7: 0x37fd, 0x1cf8: 0x3804, 0x1cf9: 0x380b, 0x1cfa: 0x3812, 0x1cfb: 0x3819, - 0x1cfc: 0x3812, 0x1cfd: 0x3804, 0x1cfe: 0x3820, 0x1cff: 0x3827, - // Block 0x74, offset 0x1d00 - 0x1d00: 0x382e, 0x1d01: 0x3835, 0x1d02: 0x383c, 0x1d03: 0x3819, 0x1d04: 0x367c, 0x1d05: 0x3636, - 0x1d06: 0x3843, 0x1d07: 0x384a, - 0x1d30: 0x3851, 0x1d31: 0x3858, 0x1d32: 0x385f, 0x1d33: 0x3868, 0x1d34: 0x3871, 0x1d35: 0x387a, - 0x1d36: 0x3883, 0x1d37: 0x388c, 0x1d38: 0x3895, 0x1d39: 0x389e, 0x1d3a: 0x38a5, 0x1d3b: 0x38c7, - 0x1d3c: 0x38d7, - // Block 0x75, offset 0x1d40 - 0x1d50: 0x38e0, 0x1d51: 0x38e2, - 0x1d52: 0x38e6, 0x1d53: 0x38ea, 0x1d54: 0x04e1, 0x1d55: 0x38ec, 0x1d56: 0x38ee, 0x1d57: 0x38f0, - 0x1d58: 0x38f4, 0x1d59: 0x1443, - 0x1d70: 0x1440, 0x1d71: 0x38f8, 0x1d72: 0x38fc, 0x1d73: 0x3900, 0x1d74: 0x3900, 0x1d75: 0x149c, - 0x1d76: 0x149e, 0x1d77: 0x3902, 0x1d78: 0x3904, 0x1d79: 0x3906, 0x1d7a: 0x390a, 0x1d7b: 0x390e, - 0x1d7c: 0x3912, 0x1d7d: 0x3916, 0x1d7e: 0x391a, 0x1d7f: 0x16c3, - // Block 0x76, offset 0x1d80 - 0x1d80: 0x16c7, 0x1d81: 0x391e, 0x1d82: 0x3922, 0x1d83: 0x3926, 0x1d84: 0x392a, - 0x1d87: 0x392e, 0x1d88: 0x3930, 0x1d89: 0x146c, 0x1d8a: 0x146c, 0x1d8b: 0x146c, - 0x1d8c: 0x146c, 0x1d8d: 0x3900, 0x1d8e: 0x3900, 0x1d8f: 0x3900, 0x1d90: 0x38e0, 0x1d91: 0x38e2, - 0x1d92: 0x143e, 0x1d94: 0x04e1, 0x1d95: 0x38ea, 0x1d96: 0x38ee, 0x1d97: 0x38ec, - 0x1d98: 0x38f8, 0x1d99: 0x149c, 0x1d9a: 0x149e, 0x1d9b: 0x3902, 0x1d9c: 0x3904, 0x1d9d: 0x3906, - 0x1d9e: 0x390a, 0x1d9f: 0x3932, 0x1da0: 0x3934, 0x1da1: 0x3936, 0x1da2: 0x1494, 0x1da3: 0x3938, - 0x1da4: 0x393a, 0x1da5: 0x393c, 0x1da6: 0x149a, 0x1da8: 0x393e, 0x1da9: 0x3940, - 0x1daa: 0x3942, 0x1dab: 0x3944, - 0x1db0: 0x3946, 0x1db1: 0x394a, 0x1db2: 0x394f, 0x1db4: 0x3953, - 0x1db6: 0x3957, 0x1db7: 0x395b, 0x1db8: 0x3960, 0x1db9: 0x3964, 0x1dba: 0x3969, 0x1dbb: 0x396d, - 0x1dbc: 0x3972, 0x1dbd: 0x3976, 0x1dbe: 0x397b, 0x1dbf: 0x397f, - // Block 0x77, offset 0x1dc0 - 0x1dc0: 0x3984, 0x1dc1: 0x068d, 0x1dc2: 0x068d, 0x1dc3: 0x0692, 0x1dc4: 0x0692, 0x1dc5: 0x0697, - 0x1dc6: 0x0697, 0x1dc7: 0x069c, 0x1dc8: 0x069c, 0x1dc9: 0x06a1, 0x1dca: 0x06a1, 0x1dcb: 0x06a1, - 0x1dcc: 0x06a1, 0x1dcd: 0x3987, 0x1dce: 0x3987, 0x1dcf: 0x398a, 0x1dd0: 0x398a, 0x1dd1: 0x398a, - 0x1dd2: 0x398a, 0x1dd3: 0x398d, 0x1dd4: 0x398d, 0x1dd5: 0x3990, 0x1dd6: 0x3990, 0x1dd7: 0x3990, - 0x1dd8: 0x3990, 0x1dd9: 0x3993, 0x1dda: 0x3993, 0x1ddb: 0x3993, 0x1ddc: 0x3993, 0x1ddd: 0x3996, - 0x1dde: 0x3996, 0x1ddf: 0x3996, 0x1de0: 0x3996, 0x1de1: 0x3999, 0x1de2: 0x3999, 0x1de3: 0x3999, - 0x1de4: 0x3999, 0x1de5: 0x399c, 0x1de6: 0x399c, 0x1de7: 0x399c, 0x1de8: 0x399c, 0x1de9: 0x399f, - 0x1dea: 0x399f, 0x1deb: 0x39a2, 0x1dec: 0x39a2, 0x1ded: 0x39a5, 0x1dee: 0x39a5, 0x1def: 0x39a8, - 0x1df0: 0x39a8, 0x1df1: 0x39ab, 0x1df2: 0x39ab, 0x1df3: 0x39ab, 0x1df4: 0x39ab, 0x1df5: 0x39ae, - 0x1df6: 0x39ae, 0x1df7: 0x39ae, 0x1df8: 0x39ae, 0x1df9: 0x39b1, 0x1dfa: 0x39b1, 0x1dfb: 0x39b1, - 0x1dfc: 0x39b1, 0x1dfd: 0x39b4, 0x1dfe: 0x39b4, 0x1dff: 0x39b4, - // Block 0x78, offset 0x1e00 - 0x1e00: 0x39b4, 0x1e01: 0x39b7, 0x1e02: 0x39b7, 0x1e03: 0x39b7, 0x1e04: 0x39b7, 0x1e05: 0x39ba, - 0x1e06: 0x39ba, 0x1e07: 0x39ba, 0x1e08: 0x39ba, 0x1e09: 0x39bd, 0x1e0a: 0x39bd, 0x1e0b: 0x39bd, - 0x1e0c: 0x39bd, 0x1e0d: 0x39c0, 0x1e0e: 0x39c0, 0x1e0f: 0x39c0, 0x1e10: 0x39c0, 0x1e11: 0x39c3, - 0x1e12: 0x39c3, 0x1e13: 0x39c3, 0x1e14: 0x39c3, 0x1e15: 0x39c6, 0x1e16: 0x39c6, 0x1e17: 0x39c6, - 0x1e18: 0x39c6, 0x1e19: 0x39c9, 0x1e1a: 0x39c9, 0x1e1b: 0x39c9, 0x1e1c: 0x39c9, 0x1e1d: 0x39cc, - 0x1e1e: 0x39cc, 0x1e1f: 0x39cc, 0x1e20: 0x39cc, 0x1e21: 0x39cf, 0x1e22: 0x39cf, 0x1e23: 0x39cf, - 0x1e24: 0x39cf, 0x1e25: 0x39d2, 0x1e26: 0x39d2, 0x1e27: 0x39d2, 0x1e28: 0x39d2, 0x1e29: 0x39d5, - 0x1e2a: 0x39d5, 0x1e2b: 0x39d5, 0x1e2c: 0x39d5, 0x1e2d: 0x39d8, 0x1e2e: 0x39d8, 0x1e2f: 0x3239, - 0x1e30: 0x3239, 0x1e31: 0x39db, 0x1e32: 0x39db, 0x1e33: 0x39db, 0x1e34: 0x39db, 0x1e35: 0x39de, - 0x1e36: 0x39de, 0x1e37: 0x39e5, 0x1e38: 0x39e5, 0x1e39: 0x39ec, 0x1e3a: 0x39ec, 0x1e3b: 0x39f3, - 0x1e3c: 0x39f3, - // Block 0x79, offset 0x1e40 - 0x1e41: 0x38ec, 0x1e42: 0x39f8, 0x1e43: 0x3932, 0x1e44: 0x3940, 0x1e45: 0x3942, - 0x1e46: 0x3934, 0x1e47: 0x39fa, 0x1e48: 0x149c, 0x1e49: 0x149e, 0x1e4a: 0x3936, 0x1e4b: 0x1494, - 0x1e4c: 0x38e0, 0x1e4d: 0x3938, 0x1e4e: 0x143e, 0x1e4f: 0x39fc, 0x1e50: 0x1486, 0x1e51: 0x001c, - 0x1e52: 0x000d, 0x1e53: 0x000f, 0x1e54: 0x1488, 0x1e55: 0x148a, 0x1e56: 0x148c, 0x1e57: 0x148e, - 0x1e58: 0x1490, 0x1e59: 0x1492, 0x1e5a: 0x38ea, 0x1e5b: 0x04e1, 0x1e5c: 0x393a, 0x1e5d: 0x149a, - 0x1e5e: 0x393c, 0x1e5f: 0x38ee, 0x1e60: 0x3944, 0x1e61: 0x0906, 0x1e62: 0x090b, 0x1e63: 0x14ad, - 0x1e64: 0x090d, 0x1e65: 0x090f, 0x1e66: 0x14d9, 0x1e67: 0x0914, 0x1e68: 0x0916, 0x1e69: 0x0918, - 0x1e6a: 0x091a, 0x1e6b: 0x091c, 0x1e6c: 0x091e, 0x1e6d: 0x0920, 0x1e6e: 0x0922, 0x1e6f: 0x0924, - 0x1e70: 0x0929, 0x1e71: 0x14c8, 0x1e72: 0x092b, 0x1e73: 0x17f6, 0x1e74: 0x092d, 0x1e75: 0x092f, - 0x1e76: 0x155f, 0x1e77: 0x0931, 0x1e78: 0x1570, 0x1e79: 0x17f8, 0x1e7a: 0x14d4, 0x1e7b: 0x392e, - 0x1e7c: 0x393e, 0x1e7d: 0x3930, 0x1e7e: 0x39fe, 0x1e7f: 0x3900, - // Block 0x7a, offset 0x1e80 - 0x1e80: 0x13f7, 0x1e81: 0x0007, 0x1e82: 0x093d, 0x1e83: 0x0984, 0x1e84: 0x093f, 0x1e85: 0x0941, - 0x1e86: 0x098c, 0x1e87: 0x094c, 0x1e88: 0x0494, 0x1e89: 0x097c, 0x1e8a: 0x0499, 0x1e8b: 0x094e, - 0x1e8c: 0x04c5, 0x1e8d: 0x0950, 0x1e8e: 0x14a0, 0x1e8f: 0x001e, 0x1e90: 0x0960, 0x1e91: 0x17fa, - 0x1e92: 0x049b, 0x1e93: 0x02c8, 0x1e94: 0x0962, 0x1e95: 0x0964, 0x1e96: 0x096d, 0x1e97: 0x04a6, - 0x1e98: 0x04c7, 0x1e99: 0x04a8, 0x1e9a: 0x09df, 0x1e9b: 0x3902, 0x1e9c: 0x3a00, 0x1e9d: 0x3904, - 0x1e9e: 0x3a02, 0x1e9f: 0x3a04, 0x1ea0: 0x3a08, 0x1ea1: 0x38e6, 0x1ea2: 0x391e, 0x1ea3: 0x3922, - 0x1ea4: 0x38e2, 0x1ea5: 0x3a0c, 0x1ea6: 0x2321, 0x1ea7: 0x3a10, 0x1ea8: 0x3a14, 0x1ea9: 0x3a18, - 0x1eaa: 0x3a1c, 0x1eab: 0x3a20, 0x1eac: 0x3a24, 0x1ead: 0x3a28, 0x1eae: 0x3a2c, 0x1eaf: 0x3a30, - 0x1eb0: 0x3a34, 0x1eb1: 0x2269, 0x1eb2: 0x226d, 0x1eb3: 0x2271, 0x1eb4: 0x2275, 0x1eb5: 0x2279, - 0x1eb6: 0x227d, 0x1eb7: 0x2281, 0x1eb8: 0x2285, 0x1eb9: 0x2289, 0x1eba: 0x228d, 0x1ebb: 0x2291, - 0x1ebc: 0x2295, 0x1ebd: 0x2299, 0x1ebe: 0x229d, 0x1ebf: 0x22a1, - // Block 0x7b, offset 0x1ec0 - 0x1ec0: 0x22a5, 0x1ec1: 0x22a9, 0x1ec2: 0x22ad, 0x1ec3: 0x22b1, 0x1ec4: 0x22b5, 0x1ec5: 0x22b9, - 0x1ec6: 0x22bd, 0x1ec7: 0x22c1, 0x1ec8: 0x22c5, 0x1ec9: 0x22c9, 0x1eca: 0x22cd, 0x1ecb: 0x22d1, - 0x1ecc: 0x22d5, 0x1ecd: 0x22d9, 0x1ece: 0x22dd, 0x1ecf: 0x22e1, 0x1ed0: 0x22e5, 0x1ed1: 0x22e9, - 0x1ed2: 0x22ed, 0x1ed3: 0x22f1, 0x1ed4: 0x22f5, 0x1ed5: 0x22f9, 0x1ed6: 0x22fd, 0x1ed7: 0x2301, - 0x1ed8: 0x2305, 0x1ed9: 0x2309, 0x1eda: 0x230d, 0x1edb: 0x2311, 0x1edc: 0x2315, 0x1edd: 0x3a38, - 0x1ede: 0x3a3c, 0x1edf: 0x3a40, 0x1ee0: 0x1e04, 0x1ee1: 0x1d38, 0x1ee2: 0x1d3c, 0x1ee3: 0x1d40, - 0x1ee4: 0x1d44, 0x1ee5: 0x1d48, 0x1ee6: 0x1d4c, 0x1ee7: 0x1d50, 0x1ee8: 0x1d54, 0x1ee9: 0x1d58, - 0x1eea: 0x1d5c, 0x1eeb: 0x1d60, 0x1eec: 0x1d64, 0x1eed: 0x1d68, 0x1eee: 0x1d6c, 0x1eef: 0x1d70, - 0x1ef0: 0x1d74, 0x1ef1: 0x1d78, 0x1ef2: 0x1d7c, 0x1ef3: 0x1d80, 0x1ef4: 0x1d84, 0x1ef5: 0x1d88, - 0x1ef6: 0x1d8c, 0x1ef7: 0x1d90, 0x1ef8: 0x1d94, 0x1ef9: 0x1d98, 0x1efa: 0x1d9c, 0x1efb: 0x1da0, - 0x1efc: 0x1da4, 0x1efd: 0x1da8, 0x1efe: 0x1dac, - // Block 0x7c, offset 0x1f00 - 0x1f02: 0x1db0, 0x1f03: 0x1db4, 0x1f04: 0x1db8, 0x1f05: 0x1dbc, - 0x1f06: 0x1dc0, 0x1f07: 0x1dc4, 0x1f0a: 0x1dc8, 0x1f0b: 0x1dcc, - 0x1f0c: 0x1dd0, 0x1f0d: 0x1dd4, 0x1f0e: 0x1dd8, 0x1f0f: 0x1ddc, - 0x1f12: 0x1de0, 0x1f13: 0x1de4, 0x1f14: 0x1de8, 0x1f15: 0x1dec, 0x1f16: 0x1df0, 0x1f17: 0x1df4, - 0x1f1a: 0x1df8, 0x1f1b: 0x1dfc, 0x1f1c: 0x1e00, - 0x1f20: 0x3a44, 0x1f21: 0x3a47, 0x1f22: 0x3a4a, 0x1f23: 0x0009, - 0x1f24: 0x3a4d, 0x1f25: 0x3a50, 0x1f26: 0x3a53, 0x1f28: 0x3a57, 0x1f29: 0x3a5b, - 0x1f2a: 0x3a5f, 0x1f2b: 0x3a63, 0x1f2c: 0x3a67, 0x1f2d: 0x3a6b, 0x1f2e: 0x3a6f, - // Block 0x7d, offset 0x1f40 - 0x1f5a: 0x3a73, 0x1f5c: 0x3a7c, - 0x1f6b: 0x3a85, - // Block 0x7e, offset 0x1f80 - 0x1f9e: 0x3a8e, 0x1f9f: 0x3a97, 0x1fa0: 0x3aa0, 0x1fa1: 0x3aad, 0x1fa2: 0x3aba, 0x1fa3: 0x3ac7, - 0x1fa4: 0x3ad4, - // Block 0x7f, offset 0x1fc0 - 0x1ffb: 0x3ae1, - 0x1ffc: 0x3aea, 0x1ffd: 0x3af3, 0x1ffe: 0x3b00, 0x1fff: 0x3b0d, - // Block 0x80, offset 0x2000 - 0x2000: 0x3b1a, - // Block 0x81, offset 0x2040 - 0x2040: 0x0906, 0x2041: 0x090b, 0x2042: 0x14ad, 0x2043: 0x090d, 0x2044: 0x090f, 0x2045: 0x14d9, - 0x2046: 0x0914, 0x2047: 0x0916, 0x2048: 0x0918, 0x2049: 0x091a, 0x204a: 0x091c, 0x204b: 0x091e, - 0x204c: 0x0920, 0x204d: 0x0922, 0x204e: 0x0924, 0x204f: 0x0929, 0x2050: 0x14c8, 0x2051: 0x092b, - 0x2052: 0x17f6, 0x2053: 0x092d, 0x2054: 0x092f, 0x2055: 0x155f, 0x2056: 0x0931, 0x2057: 0x1570, - 0x2058: 0x17f8, 0x2059: 0x14d4, 0x205a: 0x0007, 0x205b: 0x093d, 0x205c: 0x0984, 0x205d: 0x093f, - 0x205e: 0x0941, 0x205f: 0x098c, 0x2060: 0x094c, 0x2061: 0x0494, 0x2062: 0x097c, 0x2063: 0x0499, - 0x2064: 0x094e, 0x2065: 0x04c5, 0x2066: 0x0950, 0x2067: 0x14a0, 0x2068: 0x001e, 0x2069: 0x0960, - 0x206a: 0x17fa, 0x206b: 0x049b, 0x206c: 0x02c8, 0x206d: 0x0962, 0x206e: 0x0964, 0x206f: 0x096d, - 0x2070: 0x04a6, 0x2071: 0x04c7, 0x2072: 0x04a8, 0x2073: 0x09df, 0x2074: 0x0906, 0x2075: 0x090b, - 0x2076: 0x14ad, 0x2077: 0x090d, 0x2078: 0x090f, 0x2079: 0x14d9, 0x207a: 0x0914, 0x207b: 0x0916, - 0x207c: 0x0918, 0x207d: 0x091a, 0x207e: 0x091c, 0x207f: 0x091e, - // Block 0x82, offset 0x2080 - 0x2080: 0x0920, 0x2081: 0x0922, 0x2082: 0x0924, 0x2083: 0x0929, 0x2084: 0x14c8, 0x2085: 0x092b, - 0x2086: 0x17f6, 0x2087: 0x092d, 0x2088: 0x092f, 0x2089: 0x155f, 0x208a: 0x0931, 0x208b: 0x1570, - 0x208c: 0x17f8, 0x208d: 0x14d4, 0x208e: 0x0007, 0x208f: 0x093d, 0x2090: 0x0984, 0x2091: 0x093f, - 0x2092: 0x0941, 0x2093: 0x098c, 0x2094: 0x094c, 0x2096: 0x097c, 0x2097: 0x0499, - 0x2098: 0x094e, 0x2099: 0x04c5, 0x209a: 0x0950, 0x209b: 0x14a0, 0x209c: 0x001e, 0x209d: 0x0960, - 0x209e: 0x17fa, 0x209f: 0x049b, 0x20a0: 0x02c8, 0x20a1: 0x0962, 0x20a2: 0x0964, 0x20a3: 0x096d, - 0x20a4: 0x04a6, 0x20a5: 0x04c7, 0x20a6: 0x04a8, 0x20a7: 0x09df, 0x20a8: 0x0906, 0x20a9: 0x090b, - 0x20aa: 0x14ad, 0x20ab: 0x090d, 0x20ac: 0x090f, 0x20ad: 0x14d9, 0x20ae: 0x0914, 0x20af: 0x0916, - 0x20b0: 0x0918, 0x20b1: 0x091a, 0x20b2: 0x091c, 0x20b3: 0x091e, 0x20b4: 0x0920, 0x20b5: 0x0922, - 0x20b6: 0x0924, 0x20b7: 0x0929, 0x20b8: 0x14c8, 0x20b9: 0x092b, 0x20ba: 0x17f6, 0x20bb: 0x092d, - 0x20bc: 0x092f, 0x20bd: 0x155f, 0x20be: 0x0931, 0x20bf: 0x1570, - // Block 0x83, offset 0x20c0 - 0x20c0: 0x17f8, 0x20c1: 0x14d4, 0x20c2: 0x0007, 0x20c3: 0x093d, 0x20c4: 0x0984, 0x20c5: 0x093f, - 0x20c6: 0x0941, 0x20c7: 0x098c, 0x20c8: 0x094c, 0x20c9: 0x0494, 0x20ca: 0x097c, 0x20cb: 0x0499, - 0x20cc: 0x094e, 0x20cd: 0x04c5, 0x20ce: 0x0950, 0x20cf: 0x14a0, 0x20d0: 0x001e, 0x20d1: 0x0960, - 0x20d2: 0x17fa, 0x20d3: 0x049b, 0x20d4: 0x02c8, 0x20d5: 0x0962, 0x20d6: 0x0964, 0x20d7: 0x096d, - 0x20d8: 0x04a6, 0x20d9: 0x04c7, 0x20da: 0x04a8, 0x20db: 0x09df, 0x20dc: 0x0906, - 0x20de: 0x14ad, 0x20df: 0x090d, 0x20e2: 0x0914, - 0x20e5: 0x091a, 0x20e6: 0x091c, 0x20e9: 0x0922, - 0x20ea: 0x0924, 0x20eb: 0x0929, 0x20ec: 0x14c8, 0x20ee: 0x17f6, 0x20ef: 0x092d, - 0x20f0: 0x092f, 0x20f1: 0x155f, 0x20f2: 0x0931, 0x20f3: 0x1570, 0x20f4: 0x17f8, 0x20f5: 0x14d4, - 0x20f6: 0x0007, 0x20f7: 0x093d, 0x20f8: 0x0984, 0x20f9: 0x093f, 0x20fb: 0x098c, - 0x20fd: 0x0494, 0x20fe: 0x097c, 0x20ff: 0x0499, - // Block 0x84, offset 0x2100 - 0x2100: 0x094e, 0x2101: 0x04c5, 0x2102: 0x0950, 0x2103: 0x14a0, 0x2105: 0x0960, - 0x2106: 0x17fa, 0x2107: 0x049b, 0x2108: 0x02c8, 0x2109: 0x0962, 0x210a: 0x0964, 0x210b: 0x096d, - 0x210c: 0x04a6, 0x210d: 0x04c7, 0x210e: 0x04a8, 0x210f: 0x09df, 0x2110: 0x0906, 0x2111: 0x090b, - 0x2112: 0x14ad, 0x2113: 0x090d, 0x2114: 0x090f, 0x2115: 0x14d9, 0x2116: 0x0914, 0x2117: 0x0916, - 0x2118: 0x0918, 0x2119: 0x091a, 0x211a: 0x091c, 0x211b: 0x091e, 0x211c: 0x0920, 0x211d: 0x0922, - 0x211e: 0x0924, 0x211f: 0x0929, 0x2120: 0x14c8, 0x2121: 0x092b, 0x2122: 0x17f6, 0x2123: 0x092d, - 0x2124: 0x092f, 0x2125: 0x155f, 0x2126: 0x0931, 0x2127: 0x1570, 0x2128: 0x17f8, 0x2129: 0x14d4, - 0x212a: 0x0007, 0x212b: 0x093d, 0x212c: 0x0984, 0x212d: 0x093f, 0x212e: 0x0941, 0x212f: 0x098c, - 0x2130: 0x094c, 0x2131: 0x0494, 0x2132: 0x097c, 0x2133: 0x0499, 0x2134: 0x094e, 0x2135: 0x04c5, - 0x2136: 0x0950, 0x2137: 0x14a0, 0x2138: 0x001e, 0x2139: 0x0960, 0x213a: 0x17fa, 0x213b: 0x049b, - 0x213c: 0x02c8, 0x213d: 0x0962, 0x213e: 0x0964, 0x213f: 0x096d, - // Block 0x85, offset 0x2140 - 0x2140: 0x04a6, 0x2141: 0x04c7, 0x2142: 0x04a8, 0x2143: 0x09df, 0x2144: 0x0906, 0x2145: 0x090b, - 0x2147: 0x090d, 0x2148: 0x090f, 0x2149: 0x14d9, 0x214a: 0x0914, - 0x214d: 0x091a, 0x214e: 0x091c, 0x214f: 0x091e, 0x2150: 0x0920, 0x2151: 0x0922, - 0x2152: 0x0924, 0x2153: 0x0929, 0x2154: 0x14c8, 0x2156: 0x17f6, 0x2157: 0x092d, - 0x2158: 0x092f, 0x2159: 0x155f, 0x215a: 0x0931, 0x215b: 0x1570, 0x215c: 0x17f8, - 0x215e: 0x0007, 0x215f: 0x093d, 0x2160: 0x0984, 0x2161: 0x093f, 0x2162: 0x0941, 0x2163: 0x098c, - 0x2164: 0x094c, 0x2165: 0x0494, 0x2166: 0x097c, 0x2167: 0x0499, 0x2168: 0x094e, 0x2169: 0x04c5, - 0x216a: 0x0950, 0x216b: 0x14a0, 0x216c: 0x001e, 0x216d: 0x0960, 0x216e: 0x17fa, 0x216f: 0x049b, - 0x2170: 0x02c8, 0x2171: 0x0962, 0x2172: 0x0964, 0x2173: 0x096d, 0x2174: 0x04a6, 0x2175: 0x04c7, - 0x2176: 0x04a8, 0x2177: 0x09df, 0x2178: 0x0906, 0x2179: 0x090b, 0x217b: 0x090d, - 0x217c: 0x090f, 0x217d: 0x14d9, 0x217e: 0x0914, - // Block 0x86, offset 0x2180 - 0x2180: 0x0918, 0x2181: 0x091a, 0x2182: 0x091c, 0x2183: 0x091e, 0x2184: 0x0920, - 0x2186: 0x0924, 0x218a: 0x17f6, 0x218b: 0x092d, - 0x218c: 0x092f, 0x218d: 0x155f, 0x218e: 0x0931, 0x218f: 0x1570, 0x2190: 0x17f8, - 0x2192: 0x0007, 0x2193: 0x093d, 0x2194: 0x0984, 0x2195: 0x093f, 0x2196: 0x0941, 0x2197: 0x098c, - 0x2198: 0x094c, 0x2199: 0x0494, 0x219a: 0x097c, 0x219b: 0x0499, 0x219c: 0x094e, 0x219d: 0x04c5, - 0x219e: 0x0950, 0x219f: 0x14a0, 0x21a0: 0x001e, 0x21a1: 0x0960, 0x21a2: 0x17fa, 0x21a3: 0x049b, - 0x21a4: 0x02c8, 0x21a5: 0x0962, 0x21a6: 0x0964, 0x21a7: 0x096d, 0x21a8: 0x04a6, 0x21a9: 0x04c7, - 0x21aa: 0x04a8, 0x21ab: 0x09df, 0x21ac: 0x0906, 0x21ad: 0x090b, 0x21ae: 0x14ad, 0x21af: 0x090d, - 0x21b0: 0x090f, 0x21b1: 0x14d9, 0x21b2: 0x0914, 0x21b3: 0x0916, 0x21b4: 0x0918, 0x21b5: 0x091a, - 0x21b6: 0x091c, 0x21b7: 0x091e, 0x21b8: 0x0920, 0x21b9: 0x0922, 0x21ba: 0x0924, 0x21bb: 0x0929, - 0x21bc: 0x14c8, 0x21bd: 0x092b, 0x21be: 0x17f6, 0x21bf: 0x092d, - // Block 0x87, offset 0x21c0 - 0x21c0: 0x092f, 0x21c1: 0x155f, 0x21c2: 0x0931, 0x21c3: 0x1570, 0x21c4: 0x17f8, 0x21c5: 0x14d4, - 0x21c6: 0x0007, 0x21c7: 0x093d, 0x21c8: 0x0984, 0x21c9: 0x093f, 0x21ca: 0x0941, 0x21cb: 0x098c, - 0x21cc: 0x094c, 0x21cd: 0x0494, 0x21ce: 0x097c, 0x21cf: 0x0499, 0x21d0: 0x094e, 0x21d1: 0x04c5, - 0x21d2: 0x0950, 0x21d3: 0x14a0, 0x21d4: 0x001e, 0x21d5: 0x0960, 0x21d6: 0x17fa, 0x21d7: 0x049b, - 0x21d8: 0x02c8, 0x21d9: 0x0962, 0x21da: 0x0964, 0x21db: 0x096d, 0x21dc: 0x04a6, 0x21dd: 0x04c7, - 0x21de: 0x04a8, 0x21df: 0x09df, 0x21e0: 0x0906, 0x21e1: 0x090b, 0x21e2: 0x14ad, 0x21e3: 0x090d, - 0x21e4: 0x090f, 0x21e5: 0x14d9, 0x21e6: 0x0914, 0x21e7: 0x0916, 0x21e8: 0x0918, 0x21e9: 0x091a, - 0x21ea: 0x091c, 0x21eb: 0x091e, 0x21ec: 0x0920, 0x21ed: 0x0922, 0x21ee: 0x0924, 0x21ef: 0x0929, - 0x21f0: 0x14c8, 0x21f1: 0x092b, 0x21f2: 0x17f6, 0x21f3: 0x092d, 0x21f4: 0x092f, 0x21f5: 0x155f, - 0x21f6: 0x0931, 0x21f7: 0x1570, 0x21f8: 0x17f8, 0x21f9: 0x14d4, 0x21fa: 0x0007, 0x21fb: 0x093d, - 0x21fc: 0x0984, 0x21fd: 0x093f, 0x21fe: 0x0941, 0x21ff: 0x098c, - // Block 0x88, offset 0x2200 - 0x2200: 0x094c, 0x2201: 0x0494, 0x2202: 0x097c, 0x2203: 0x0499, 0x2204: 0x094e, 0x2205: 0x04c5, - 0x2206: 0x0950, 0x2207: 0x14a0, 0x2208: 0x001e, 0x2209: 0x0960, 0x220a: 0x17fa, 0x220b: 0x049b, - 0x220c: 0x02c8, 0x220d: 0x0962, 0x220e: 0x0964, 0x220f: 0x096d, 0x2210: 0x04a6, 0x2211: 0x04c7, - 0x2212: 0x04a8, 0x2213: 0x09df, 0x2214: 0x0906, 0x2215: 0x090b, 0x2216: 0x14ad, 0x2217: 0x090d, - 0x2218: 0x090f, 0x2219: 0x14d9, 0x221a: 0x0914, 0x221b: 0x0916, 0x221c: 0x0918, 0x221d: 0x091a, - 0x221e: 0x091c, 0x221f: 0x091e, 0x2220: 0x0920, 0x2221: 0x0922, 0x2222: 0x0924, 0x2223: 0x0929, - 0x2224: 0x14c8, 0x2225: 0x092b, 0x2226: 0x17f6, 0x2227: 0x092d, 0x2228: 0x092f, 0x2229: 0x155f, - 0x222a: 0x0931, 0x222b: 0x1570, 0x222c: 0x17f8, 0x222d: 0x14d4, 0x222e: 0x0007, 0x222f: 0x093d, - 0x2230: 0x0984, 0x2231: 0x093f, 0x2232: 0x0941, 0x2233: 0x098c, 0x2234: 0x094c, 0x2235: 0x0494, - 0x2236: 0x097c, 0x2237: 0x0499, 0x2238: 0x094e, 0x2239: 0x04c5, 0x223a: 0x0950, 0x223b: 0x14a0, - 0x223c: 0x001e, 0x223d: 0x0960, 0x223e: 0x17fa, 0x223f: 0x049b, - // Block 0x89, offset 0x2240 - 0x2240: 0x02c8, 0x2241: 0x0962, 0x2242: 0x0964, 0x2243: 0x096d, 0x2244: 0x04a6, 0x2245: 0x04c7, - 0x2246: 0x04a8, 0x2247: 0x09df, 0x2248: 0x0906, 0x2249: 0x090b, 0x224a: 0x14ad, 0x224b: 0x090d, - 0x224c: 0x090f, 0x224d: 0x14d9, 0x224e: 0x0914, 0x224f: 0x0916, 0x2250: 0x0918, 0x2251: 0x091a, - 0x2252: 0x091c, 0x2253: 0x091e, 0x2254: 0x0920, 0x2255: 0x0922, 0x2256: 0x0924, 0x2257: 0x0929, - 0x2258: 0x14c8, 0x2259: 0x092b, 0x225a: 0x17f6, 0x225b: 0x092d, 0x225c: 0x092f, 0x225d: 0x155f, - 0x225e: 0x0931, 0x225f: 0x1570, 0x2260: 0x17f8, 0x2261: 0x14d4, 0x2262: 0x0007, 0x2263: 0x093d, - 0x2264: 0x0984, 0x2265: 0x093f, 0x2266: 0x0941, 0x2267: 0x098c, 0x2268: 0x094c, 0x2269: 0x0494, - 0x226a: 0x097c, 0x226b: 0x0499, 0x226c: 0x094e, 0x226d: 0x04c5, 0x226e: 0x0950, 0x226f: 0x14a0, - 0x2270: 0x001e, 0x2271: 0x0960, 0x2272: 0x17fa, 0x2273: 0x049b, 0x2274: 0x02c8, 0x2275: 0x0962, - 0x2276: 0x0964, 0x2277: 0x096d, 0x2278: 0x04a6, 0x2279: 0x04c7, 0x227a: 0x04a8, 0x227b: 0x09df, - 0x227c: 0x0906, 0x227d: 0x090b, 0x227e: 0x14ad, 0x227f: 0x090d, - // Block 0x8a, offset 0x2280 - 0x2280: 0x090f, 0x2281: 0x14d9, 0x2282: 0x0914, 0x2283: 0x0916, 0x2284: 0x0918, 0x2285: 0x091a, - 0x2286: 0x091c, 0x2287: 0x091e, 0x2288: 0x0920, 0x2289: 0x0922, 0x228a: 0x0924, 0x228b: 0x0929, - 0x228c: 0x14c8, 0x228d: 0x092b, 0x228e: 0x17f6, 0x228f: 0x092d, 0x2290: 0x092f, 0x2291: 0x155f, - 0x2292: 0x0931, 0x2293: 0x1570, 0x2294: 0x17f8, 0x2295: 0x14d4, 0x2296: 0x0007, 0x2297: 0x093d, - 0x2298: 0x0984, 0x2299: 0x093f, 0x229a: 0x0941, 0x229b: 0x098c, 0x229c: 0x094c, 0x229d: 0x0494, - 0x229e: 0x097c, 0x229f: 0x0499, 0x22a0: 0x094e, 0x22a1: 0x04c5, 0x22a2: 0x0950, 0x22a3: 0x14a0, - 0x22a4: 0x001e, 0x22a5: 0x0960, 0x22a6: 0x17fa, 0x22a7: 0x049b, 0x22a8: 0x02c8, 0x22a9: 0x0962, - 0x22aa: 0x0964, 0x22ab: 0x096d, 0x22ac: 0x04a6, 0x22ad: 0x04c7, 0x22ae: 0x04a8, 0x22af: 0x09df, - 0x22b0: 0x0906, 0x22b1: 0x090b, 0x22b2: 0x14ad, 0x22b3: 0x090d, 0x22b4: 0x090f, 0x22b5: 0x14d9, - 0x22b6: 0x0914, 0x22b7: 0x0916, 0x22b8: 0x0918, 0x22b9: 0x091a, 0x22ba: 0x091c, 0x22bb: 0x091e, - 0x22bc: 0x0920, 0x22bd: 0x0922, 0x22be: 0x0924, 0x22bf: 0x0929, - // Block 0x8b, offset 0x22c0 - 0x22c0: 0x14c8, 0x22c1: 0x092b, 0x22c2: 0x17f6, 0x22c3: 0x092d, 0x22c4: 0x092f, 0x22c5: 0x155f, - 0x22c6: 0x0931, 0x22c7: 0x1570, 0x22c8: 0x17f8, 0x22c9: 0x14d4, 0x22ca: 0x0007, 0x22cb: 0x093d, - 0x22cc: 0x0984, 0x22cd: 0x093f, 0x22ce: 0x0941, 0x22cf: 0x098c, 0x22d0: 0x094c, 0x22d1: 0x0494, - 0x22d2: 0x097c, 0x22d3: 0x0499, 0x22d4: 0x094e, 0x22d5: 0x04c5, 0x22d6: 0x0950, 0x22d7: 0x14a0, - 0x22d8: 0x001e, 0x22d9: 0x0960, 0x22da: 0x17fa, 0x22db: 0x049b, 0x22dc: 0x02c8, 0x22dd: 0x0962, - 0x22de: 0x0964, 0x22df: 0x096d, 0x22e0: 0x04a6, 0x22e1: 0x04c7, 0x22e2: 0x04a8, 0x22e3: 0x09df, - 0x22e4: 0x3b27, 0x22e5: 0x3b2a, 0x22e8: 0x3b2d, 0x22e9: 0x3b30, - 0x22ea: 0x14eb, 0x22eb: 0x3b33, 0x22ec: 0x3b36, 0x22ed: 0x3b39, 0x22ee: 0x3b3c, 0x22ef: 0x057b, - 0x22f0: 0x3b3f, 0x22f1: 0x3b42, 0x22f2: 0x3b45, 0x22f3: 0x3b48, 0x22f4: 0x3b4b, 0x22f5: 0x3b4e, - 0x22f6: 0x3b51, 0x22f7: 0x14ee, 0x22f8: 0x3b54, 0x22f9: 0x057b, 0x22fa: 0x0581, 0x22fb: 0x3b57, - 0x22fc: 0x055f, 0x22fd: 0x3b5a, 0x22fe: 0x3b5d, 0x22ff: 0x3b60, - // Block 0x8c, offset 0x2300 - 0x2300: 0x14d6, 0x2301: 0x3b63, 0x2302: 0x3b67, 0x2303: 0x0559, 0x2304: 0x0973, 0x2305: 0x0976, - 0x2306: 0x057e, 0x2307: 0x3b6a, 0x2308: 0x3b6d, 0x2309: 0x055c, 0x230a: 0x12fd, 0x230b: 0x0572, - 0x230c: 0x3b70, 0x230d: 0x0015, 0x230e: 0x3b73, 0x230f: 0x3b76, 0x2310: 0x3b79, 0x2311: 0x056f, - 0x2312: 0x0575, 0x2313: 0x0578, 0x2314: 0x3b7c, 0x2315: 0x3b7f, 0x2316: 0x3b82, 0x2317: 0x056c, - 0x2318: 0x0979, 0x2319: 0x3b85, 0x231a: 0x3b88, 0x231b: 0x3b8b, 0x231c: 0x057e, 0x231d: 0x055c, - 0x231e: 0x0572, 0x231f: 0x056c, 0x2320: 0x0575, 0x2321: 0x056f, 0x2322: 0x3b2d, 0x2323: 0x3b30, - 0x2324: 0x14eb, 0x2325: 0x3b33, 0x2326: 0x3b36, 0x2327: 0x3b39, 0x2328: 0x3b3c, 0x2329: 0x057b, - 0x232a: 0x3b3f, 0x232b: 0x3b42, 0x232c: 0x3b45, 0x232d: 0x3b48, 0x232e: 0x3b4b, 0x232f: 0x3b4e, - 0x2330: 0x3b51, 0x2331: 0x14ee, 0x2332: 0x3b54, 0x2333: 0x057b, 0x2334: 0x0581, 0x2335: 0x3b57, - 0x2336: 0x055f, 0x2337: 0x3b5a, 0x2338: 0x3b5d, 0x2339: 0x3b60, 0x233a: 0x14d6, 0x233b: 0x3b63, - 0x233c: 0x3b67, 0x233d: 0x0559, 0x233e: 0x0973, 0x233f: 0x0976, - // Block 0x8d, offset 0x2340 - 0x2340: 0x057e, 0x2341: 0x3b6a, 0x2342: 0x3b6d, 0x2343: 0x055c, 0x2344: 0x12fd, 0x2345: 0x0572, - 0x2346: 0x3b70, 0x2347: 0x0015, 0x2348: 0x3b73, 0x2349: 0x3b76, 0x234a: 0x3b79, 0x234b: 0x056f, - 0x234c: 0x0575, 0x234d: 0x0578, 0x234e: 0x3b7c, 0x234f: 0x3b7f, 0x2350: 0x3b82, 0x2351: 0x056c, - 0x2352: 0x0979, 0x2353: 0x3b85, 0x2354: 0x3b88, 0x2355: 0x3b8b, 0x2356: 0x057e, 0x2357: 0x055c, - 0x2358: 0x0572, 0x2359: 0x056c, 0x235a: 0x0575, 0x235b: 0x056f, 0x235c: 0x3b2d, 0x235d: 0x3b30, - 0x235e: 0x14eb, 0x235f: 0x3b33, 0x2360: 0x3b36, 0x2361: 0x3b39, 0x2362: 0x3b3c, 0x2363: 0x057b, - 0x2364: 0x3b3f, 0x2365: 0x3b42, 0x2366: 0x3b45, 0x2367: 0x3b48, 0x2368: 0x3b4b, 0x2369: 0x3b4e, - 0x236a: 0x3b51, 0x236b: 0x14ee, 0x236c: 0x3b54, 0x236d: 0x057b, 0x236e: 0x0581, 0x236f: 0x3b57, - 0x2370: 0x055f, 0x2371: 0x3b5a, 0x2372: 0x3b5d, 0x2373: 0x3b60, 0x2374: 0x14d6, 0x2375: 0x3b63, - 0x2376: 0x3b67, 0x2377: 0x0559, 0x2378: 0x0973, 0x2379: 0x0976, 0x237a: 0x057e, 0x237b: 0x3b6a, - 0x237c: 0x3b6d, 0x237d: 0x055c, 0x237e: 0x12fd, 0x237f: 0x0572, - // Block 0x8e, offset 0x2380 - 0x2380: 0x3b70, 0x2381: 0x0015, 0x2382: 0x3b73, 0x2383: 0x3b76, 0x2384: 0x3b79, 0x2385: 0x056f, - 0x2386: 0x0575, 0x2387: 0x0578, 0x2388: 0x3b7c, 0x2389: 0x3b7f, 0x238a: 0x3b82, 0x238b: 0x056c, - 0x238c: 0x0979, 0x238d: 0x3b85, 0x238e: 0x3b88, 0x238f: 0x3b8b, 0x2390: 0x057e, 0x2391: 0x055c, - 0x2392: 0x0572, 0x2393: 0x056c, 0x2394: 0x0575, 0x2395: 0x056f, 0x2396: 0x3b2d, 0x2397: 0x3b30, - 0x2398: 0x14eb, 0x2399: 0x3b33, 0x239a: 0x3b36, 0x239b: 0x3b39, 0x239c: 0x3b3c, 0x239d: 0x057b, - 0x239e: 0x3b3f, 0x239f: 0x3b42, 0x23a0: 0x3b45, 0x23a1: 0x3b48, 0x23a2: 0x3b4b, 0x23a3: 0x3b4e, - 0x23a4: 0x3b51, 0x23a5: 0x14ee, 0x23a6: 0x3b54, 0x23a7: 0x057b, 0x23a8: 0x0581, 0x23a9: 0x3b57, - 0x23aa: 0x055f, 0x23ab: 0x3b5a, 0x23ac: 0x3b5d, 0x23ad: 0x3b60, 0x23ae: 0x14d6, 0x23af: 0x3b63, - 0x23b0: 0x3b67, 0x23b1: 0x0559, 0x23b2: 0x0973, 0x23b3: 0x0976, 0x23b4: 0x057e, 0x23b5: 0x3b6a, - 0x23b6: 0x3b6d, 0x23b7: 0x055c, 0x23b8: 0x12fd, 0x23b9: 0x0572, 0x23ba: 0x3b70, 0x23bb: 0x0015, - 0x23bc: 0x3b73, 0x23bd: 0x3b76, 0x23be: 0x3b79, 0x23bf: 0x056f, - // Block 0x8f, offset 0x23c0 - 0x23c0: 0x0575, 0x23c1: 0x0578, 0x23c2: 0x3b7c, 0x23c3: 0x3b7f, 0x23c4: 0x3b82, 0x23c5: 0x056c, - 0x23c6: 0x0979, 0x23c7: 0x3b85, 0x23c8: 0x3b88, 0x23c9: 0x3b8b, 0x23ca: 0x057e, 0x23cb: 0x055c, - 0x23cc: 0x0572, 0x23cd: 0x056c, 0x23ce: 0x0575, 0x23cf: 0x056f, 0x23d0: 0x3b2d, 0x23d1: 0x3b30, - 0x23d2: 0x14eb, 0x23d3: 0x3b33, 0x23d4: 0x3b36, 0x23d5: 0x3b39, 0x23d6: 0x3b3c, 0x23d7: 0x057b, - 0x23d8: 0x3b3f, 0x23d9: 0x3b42, 0x23da: 0x3b45, 0x23db: 0x3b48, 0x23dc: 0x3b4b, 0x23dd: 0x3b4e, - 0x23de: 0x3b51, 0x23df: 0x14ee, 0x23e0: 0x3b54, 0x23e1: 0x057b, 0x23e2: 0x0581, 0x23e3: 0x3b57, - 0x23e4: 0x055f, 0x23e5: 0x3b5a, 0x23e6: 0x3b5d, 0x23e7: 0x3b60, 0x23e8: 0x14d6, 0x23e9: 0x3b63, - 0x23ea: 0x3b67, 0x23eb: 0x0559, 0x23ec: 0x0973, 0x23ed: 0x0976, 0x23ee: 0x057e, 0x23ef: 0x3b6a, - 0x23f0: 0x3b6d, 0x23f1: 0x055c, 0x23f2: 0x12fd, 0x23f3: 0x0572, 0x23f4: 0x3b70, 0x23f5: 0x0015, - 0x23f6: 0x3b73, 0x23f7: 0x3b76, 0x23f8: 0x3b79, 0x23f9: 0x056f, 0x23fa: 0x0575, 0x23fb: 0x0578, - 0x23fc: 0x3b7c, 0x23fd: 0x3b7f, 0x23fe: 0x3b82, 0x23ff: 0x056c, - // Block 0x90, offset 0x2400 - 0x2400: 0x0979, 0x2401: 0x3b85, 0x2402: 0x3b88, 0x2403: 0x3b8b, 0x2404: 0x057e, 0x2405: 0x055c, - 0x2406: 0x0572, 0x2407: 0x056c, 0x2408: 0x0575, 0x2409: 0x056f, 0x240a: 0x3b8f, 0x240b: 0x3b92, - 0x240e: 0x1486, 0x240f: 0x001c, 0x2410: 0x000d, 0x2411: 0x000f, - 0x2412: 0x1488, 0x2413: 0x148a, 0x2414: 0x148c, 0x2415: 0x148e, 0x2416: 0x1490, 0x2417: 0x1492, - 0x2418: 0x1486, 0x2419: 0x001c, 0x241a: 0x000d, 0x241b: 0x000f, 0x241c: 0x1488, 0x241d: 0x148a, - 0x241e: 0x148c, 0x241f: 0x148e, 0x2420: 0x1490, 0x2421: 0x1492, 0x2422: 0x1486, 0x2423: 0x001c, - 0x2424: 0x000d, 0x2425: 0x000f, 0x2426: 0x1488, 0x2427: 0x148a, 0x2428: 0x148c, 0x2429: 0x148e, - 0x242a: 0x1490, 0x242b: 0x1492, 0x242c: 0x1486, 0x242d: 0x001c, 0x242e: 0x000d, 0x242f: 0x000f, - 0x2430: 0x1488, 0x2431: 0x148a, 0x2432: 0x148c, 0x2433: 0x148e, 0x2434: 0x1490, 0x2435: 0x1492, - 0x2436: 0x1486, 0x2437: 0x001c, 0x2438: 0x000d, 0x2439: 0x000f, 0x243a: 0x1488, 0x243b: 0x148a, - 0x243c: 0x148c, 0x243d: 0x148e, 0x243e: 0x1490, 0x243f: 0x1492, - // Block 0x91, offset 0x2440 - 0x2440: 0x3b95, 0x2441: 0x3b98, 0x2442: 0x3b9b, 0x2443: 0x3b9e, 0x2444: 0x3ba1, 0x2445: 0x3ba4, - 0x2446: 0x3ba7, 0x2447: 0x3baa, 0x2448: 0x3bad, 0x2449: 0x3bb0, 0x244a: 0x3bb3, - 0x2450: 0x3bb6, 0x2451: 0x3bba, - 0x2452: 0x3bbe, 0x2453: 0x3bc2, 0x2454: 0x3bc6, 0x2455: 0x3bca, 0x2456: 0x3bce, 0x2457: 0x3bd2, - 0x2458: 0x3bd6, 0x2459: 0x3bda, 0x245a: 0x3bde, 0x245b: 0x3be2, 0x245c: 0x3be6, 0x245d: 0x3bea, - 0x245e: 0x3bee, 0x245f: 0x3bf2, 0x2460: 0x3bf6, 0x2461: 0x3bfa, 0x2462: 0x3bfe, 0x2463: 0x3c02, - 0x2464: 0x3c06, 0x2465: 0x3c0a, 0x2466: 0x3c0e, 0x2467: 0x3c12, 0x2468: 0x3c16, 0x2469: 0x3c1a, - 0x246a: 0x3c1e, 0x246b: 0x14ad, 0x246c: 0x092b, 0x246d: 0x3c26, 0x246e: 0x3c29, - 0x2470: 0x0906, 0x2471: 0x090b, 0x2472: 0x14ad, 0x2473: 0x090d, 0x2474: 0x090f, 0x2475: 0x14d9, - 0x2476: 0x0914, 0x2477: 0x0916, 0x2478: 0x0918, 0x2479: 0x091a, 0x247a: 0x091c, 0x247b: 0x091e, - 0x247c: 0x0920, 0x247d: 0x0922, 0x247e: 0x0924, 0x247f: 0x0929, - // Block 0x92, offset 0x2480 - 0x2480: 0x14c8, 0x2481: 0x092b, 0x2482: 0x17f6, 0x2483: 0x092d, 0x2484: 0x092f, 0x2485: 0x155f, - 0x2486: 0x0931, 0x2487: 0x1570, 0x2488: 0x17f8, 0x2489: 0x14d4, 0x248a: 0x3c2c, 0x248b: 0x293d, - 0x248c: 0x3c2f, 0x248d: 0x3c32, 0x248e: 0x3c35, 0x248f: 0x3c39, - // Block 0x93, offset 0x24c0 - 0x24d0: 0x3c3c, - // Block 0x94, offset 0x2500 - 0x2500: 0x3c3f, 0x2501: 0x3c46, 0x2502: 0x2291, - 0x2510: 0x1922, 0x2511: 0x3c4d, - 0x2512: 0x3c51, 0x2513: 0x1cb3, 0x2514: 0x183e, 0x2515: 0x3c55, 0x2516: 0x3c59, 0x2517: 0x1ed0, - 0x2518: 0x3c5d, 0x2519: 0x3c61, 0x251a: 0x3c65, 0x251b: 0x2d49, 0x251c: 0x3c69, 0x251d: 0x3c6d, - 0x251e: 0x3c71, 0x251f: 0x3c75, 0x2520: 0x3c79, 0x2521: 0x3c7d, 0x2522: 0x19b2, 0x2523: 0x3c81, - 0x2524: 0x3c85, 0x2525: 0x3c89, 0x2526: 0x3c8d, 0x2527: 0x3c91, 0x2528: 0x3c95, 0x2529: 0x1826, - 0x252a: 0x1eb0, 0x252b: 0x3c99, 0x252c: 0x21c7, 0x252d: 0x1ebc, 0x252e: 0x21cb, 0x252f: 0x3c9d, - 0x2530: 0x1a92, 0x2531: 0x3ca1, 0x2532: 0x3ca5, 0x2533: 0x3ca9, 0x2534: 0x3cad, 0x2535: 0x3cb1, - 0x2536: 0x2183, 0x2537: 0x194a, 0x2538: 0x3cb5, 0x2539: 0x3cb9, 0x253a: 0x3cbd, - // Block 0x95, offset 0x2540 - 0x2540: 0x3cc1, 0x2541: 0x3ccb, 0x2542: 0x3cd5, 0x2543: 0x3cdf, 0x2544: 0x3ce9, 0x2545: 0x3cf3, - 0x2546: 0x3cfd, 0x2547: 0x3d07, 0x2548: 0x3d11, - 0x2550: 0x3d1b, 0x2551: 0x3d1f, - // Block 0x96, offset 0x2580 - 0x2580: 0x3d23, 0x2581: 0x3d27, 0x2582: 0x3d2b, 0x2583: 0x3d2f, 0x2584: 0x3d34, 0x2585: 0x2eb5, - 0x2586: 0x3d38, 0x2587: 0x3d3c, 0x2588: 0x3d40, 0x2589: 0x3d44, 0x258a: 0x2eb9, 0x258b: 0x3d48, - 0x258c: 0x3d4c, 0x258d: 0x3d50, 0x258e: 0x2ebd, 0x258f: 0x3d55, 0x2590: 0x3d59, 0x2591: 0x3d5d, - 0x2592: 0x3d61, 0x2593: 0x3d66, 0x2594: 0x3d6a, 0x2595: 0x3c71, 0x2596: 0x3d6e, 0x2597: 0x3d73, - 0x2598: 0x3d77, 0x2599: 0x3d7b, 0x259a: 0x3d7f, 0x259b: 0x2f9a, 0x259c: 0x3d83, 0x259d: 0x1866, - 0x259e: 0x3d88, 0x259f: 0x3d8c, 0x25a0: 0x3d90, 0x25a1: 0x3d94, 0x25a2: 0x3cb9, 0x25a3: 0x3d98, - 0x25a4: 0x3d9c, 0x25a5: 0x2fae, 0x25a6: 0x2ec1, 0x25a7: 0x2ec5, 0x25a8: 0x2fb2, 0x25a9: 0x3da0, - 0x25aa: 0x3da4, 0x25ab: 0x2bf1, 0x25ac: 0x3da8, 0x25ad: 0x2ec9, 0x25ae: 0x3dac, 0x25af: 0x3db0, - 0x25b0: 0x3db4, 0x25b1: 0x3db8, 0x25b2: 0x3db8, 0x25b3: 0x3db8, 0x25b4: 0x3dbc, 0x25b5: 0x3dc1, - 0x25b6: 0x3dc5, 0x25b7: 0x3dc9, 0x25b8: 0x3dcd, 0x25b9: 0x3dd2, 0x25ba: 0x3dd6, 0x25bb: 0x3dda, - 0x25bc: 0x3dde, 0x25bd: 0x3de2, 0x25be: 0x3de6, 0x25bf: 0x3dea, - // Block 0x97, offset 0x25c0 - 0x25c0: 0x3dee, 0x25c1: 0x3df2, 0x25c2: 0x3df6, 0x25c3: 0x3dfa, 0x25c4: 0x3dfe, 0x25c5: 0x3e02, - 0x25c6: 0x3e02, 0x25c7: 0x2fba, 0x25c8: 0x3e06, 0x25c9: 0x3e0a, 0x25ca: 0x3e0e, 0x25cb: 0x3e12, - 0x25cc: 0x2ed1, 0x25cd: 0x3e16, 0x25ce: 0x3e1a, 0x25cf: 0x3e1e, 0x25d0: 0x2e39, 0x25d1: 0x3e22, - 0x25d2: 0x3e26, 0x25d3: 0x3e2a, 0x25d4: 0x3e2e, 0x25d5: 0x3e32, 0x25d6: 0x3e36, 0x25d7: 0x3e3a, - 0x25d8: 0x3e3e, 0x25d9: 0x3e42, 0x25da: 0x3e47, 0x25db: 0x3e4b, 0x25dc: 0x3e4f, 0x25dd: 0x3c55, - 0x25de: 0x3e53, 0x25df: 0x3e57, 0x25e0: 0x3e5b, 0x25e1: 0x3e60, 0x25e2: 0x3e65, 0x25e3: 0x3e69, - 0x25e4: 0x3e6d, 0x25e5: 0x3e71, 0x25e6: 0x3e75, 0x25e7: 0x3e79, 0x25e8: 0x3e7d, 0x25e9: 0x3e81, - 0x25ea: 0x3e85, 0x25eb: 0x3e85, 0x25ec: 0x3e89, 0x25ed: 0x3e8e, 0x25ee: 0x3e92, 0x25ef: 0x2be1, - 0x25f0: 0x3e96, 0x25f1: 0x3e9a, 0x25f2: 0x3e9f, 0x25f3: 0x3ea3, 0x25f4: 0x3ea7, 0x25f5: 0x18ce, - 0x25f6: 0x3eab, 0x25f7: 0x3eaf, 0x25f8: 0x18d6, 0x25f9: 0x3eb3, 0x25fa: 0x3eb7, 0x25fb: 0x3ebb, - 0x25fc: 0x3ec0, 0x25fd: 0x3ec4, 0x25fe: 0x3ec9, 0x25ff: 0x3ecd, - // Block 0x98, offset 0x2600 - 0x2600: 0x3ed1, 0x2601: 0x3ed5, 0x2602: 0x3ed9, 0x2603: 0x3edd, 0x2604: 0x3ee1, 0x2605: 0x3ee5, - 0x2606: 0x3ee9, 0x2607: 0x3eed, 0x2608: 0x3ef1, 0x2609: 0x3ef5, 0x260a: 0x3efa, 0x260b: 0x3efe, - 0x260c: 0x3f02, 0x260d: 0x3f06, 0x260e: 0x2b11, 0x260f: 0x3f0a, 0x2610: 0x18fe, 0x2611: 0x3f0f, - 0x2612: 0x3f0f, 0x2613: 0x3f14, 0x2614: 0x3f18, 0x2615: 0x3f18, 0x2616: 0x3f1c, 0x2617: 0x3f20, - 0x2618: 0x3f25, 0x2619: 0x3f2a, 0x261a: 0x3f2e, 0x261b: 0x3f32, 0x261c: 0x3f36, 0x261d: 0x3f3a, - 0x261e: 0x3f3e, 0x261f: 0x3f42, 0x2620: 0x3f46, 0x2621: 0x3f4a, 0x2622: 0x3f4e, 0x2623: 0x2ee5, - 0x2624: 0x3f52, 0x2625: 0x3f57, 0x2626: 0x3f5b, 0x2627: 0x3f5f, 0x2628: 0x2fea, 0x2629: 0x3f5f, - 0x262a: 0x3f63, 0x262b: 0x2eed, 0x262c: 0x3f67, 0x262d: 0x3f6b, 0x262e: 0x3f6f, 0x262f: 0x3f73, - 0x2630: 0x2ef1, 0x2631: 0x2aa5, 0x2632: 0x3f77, 0x2633: 0x3f7b, 0x2634: 0x3f7f, 0x2635: 0x3f83, - 0x2636: 0x3f87, 0x2637: 0x3f8b, 0x2638: 0x3f8f, 0x2639: 0x3f94, 0x263a: 0x3f98, 0x263b: 0x3f9c, - 0x263c: 0x3fa0, 0x263d: 0x3fa4, 0x263e: 0x3fa8, 0x263f: 0x3fad, - // Block 0x99, offset 0x2640 - 0x2640: 0x3fb1, 0x2641: 0x3fb5, 0x2642: 0x3fb9, 0x2643: 0x3fbd, 0x2644: 0x3fc1, 0x2645: 0x3fc5, - 0x2646: 0x3fc9, 0x2647: 0x3fcd, 0x2648: 0x2ef5, 0x2649: 0x3fd1, 0x264a: 0x3fd5, 0x264b: 0x3fda, - 0x264c: 0x3fde, 0x264d: 0x3fe2, 0x264e: 0x3fe6, 0x264f: 0x2efd, 0x2650: 0x3fea, 0x2651: 0x3fee, - 0x2652: 0x3ff2, 0x2653: 0x3ff6, 0x2654: 0x3ffa, 0x2655: 0x3ffe, 0x2656: 0x4002, 0x2657: 0x4006, - 0x2658: 0x2b15, 0x2659: 0x300a, 0x265a: 0x400a, 0x265b: 0x400e, 0x265c: 0x4012, 0x265d: 0x4016, - 0x265e: 0x401b, 0x265f: 0x401f, 0x2660: 0x4023, 0x2661: 0x4027, 0x2662: 0x2f01, 0x2663: 0x402b, - 0x2664: 0x4030, 0x2665: 0x4034, 0x2666: 0x4038, 0x2667: 0x30b5, 0x2668: 0x403c, 0x2669: 0x4040, - 0x266a: 0x4044, 0x266b: 0x4048, 0x266c: 0x404c, 0x266d: 0x4051, 0x266e: 0x4055, 0x266f: 0x4059, - 0x2670: 0x405d, 0x2671: 0x4062, 0x2672: 0x4066, 0x2673: 0x406a, 0x2674: 0x406e, 0x2675: 0x2c25, - 0x2676: 0x4072, 0x2677: 0x4076, 0x2678: 0x407b, 0x2679: 0x4080, 0x267a: 0x4085, 0x267b: 0x4089, - 0x267c: 0x408e, 0x267d: 0x4092, 0x267e: 0x4096, 0x267f: 0x409a, - // Block 0x9a, offset 0x2680 - 0x2680: 0x409e, 0x2681: 0x2f05, 0x2682: 0x2d71, 0x2683: 0x40a2, 0x2684: 0x40a6, 0x2685: 0x40aa, - 0x2686: 0x40ae, 0x2687: 0x40b3, 0x2688: 0x40b7, 0x2689: 0x40bb, 0x268a: 0x40bf, 0x268b: 0x3016, - 0x268c: 0x40c3, 0x268d: 0x40c7, 0x268e: 0x40cc, 0x268f: 0x40d0, 0x2690: 0x40d4, 0x2691: 0x40d9, - 0x2692: 0x40de, 0x2693: 0x40e2, 0x2694: 0x301a, 0x2695: 0x40e6, 0x2696: 0x40ea, 0x2697: 0x40ee, - 0x2698: 0x40f2, 0x2699: 0x40f6, 0x269a: 0x40fa, 0x269b: 0x40fe, 0x269c: 0x4103, 0x269d: 0x4107, - 0x269e: 0x410c, 0x269f: 0x4110, 0x26a0: 0x4115, 0x26a1: 0x3022, 0x26a2: 0x4119, 0x26a3: 0x411d, - 0x26a4: 0x4122, 0x26a5: 0x4126, 0x26a6: 0x412a, 0x26a7: 0x412f, 0x26a8: 0x4134, 0x26a9: 0x4138, - 0x26aa: 0x413c, 0x26ab: 0x4140, 0x26ac: 0x4144, 0x26ad: 0x4144, 0x26ae: 0x4148, 0x26af: 0x414c, - 0x26b0: 0x302a, 0x26b1: 0x4150, 0x26b2: 0x4154, 0x26b3: 0x4158, 0x26b4: 0x415c, 0x26b5: 0x4160, - 0x26b6: 0x4165, 0x26b7: 0x4169, 0x26b8: 0x2bed, 0x26b9: 0x416e, 0x26ba: 0x4173, 0x26bb: 0x4177, - 0x26bc: 0x417c, 0x26bd: 0x4181, 0x26be: 0x4186, 0x26bf: 0x418a, - // Block 0x9b, offset 0x26c0 - 0x26c0: 0x3042, 0x26c1: 0x418e, 0x26c2: 0x4193, 0x26c3: 0x4198, 0x26c4: 0x419d, 0x26c5: 0x41a2, - 0x26c6: 0x41a6, 0x26c7: 0x41a6, 0x26c8: 0x3046, 0x26c9: 0x30bd, 0x26ca: 0x41aa, 0x26cb: 0x41ae, - 0x26cc: 0x41b2, 0x26cd: 0x41b6, 0x26ce: 0x41bb, 0x26cf: 0x2b59, 0x26d0: 0x304e, 0x26d1: 0x41bf, - 0x26d2: 0x41c3, 0x26d3: 0x2f2d, 0x26d4: 0x41c8, 0x26d5: 0x41cd, 0x26d6: 0x2e89, 0x26d7: 0x41d2, - 0x26d8: 0x41d6, 0x26d9: 0x2f39, 0x26da: 0x41da, 0x26db: 0x41de, 0x26dc: 0x41e2, 0x26dd: 0x41e7, - 0x26de: 0x41e7, 0x26df: 0x41ec, 0x26e0: 0x41f0, 0x26e1: 0x41f4, 0x26e2: 0x41f9, 0x26e3: 0x41fd, - 0x26e4: 0x4201, 0x26e5: 0x4205, 0x26e6: 0x420a, 0x26e7: 0x420e, 0x26e8: 0x4212, 0x26e9: 0x4216, - 0x26ea: 0x421a, 0x26eb: 0x421e, 0x26ec: 0x4223, 0x26ed: 0x4227, 0x26ee: 0x422b, 0x26ef: 0x422f, - 0x26f0: 0x4233, 0x26f1: 0x4237, 0x26f2: 0x423b, 0x26f3: 0x4240, 0x26f4: 0x4245, 0x26f5: 0x4249, - 0x26f6: 0x424e, 0x26f7: 0x4252, 0x26f8: 0x4257, 0x26f9: 0x425b, 0x26fa: 0x2f51, 0x26fb: 0x425f, - 0x26fc: 0x4264, 0x26fd: 0x4269, 0x26fe: 0x426d, 0x26ff: 0x4272, - // Block 0x9c, offset 0x2700 - 0x2700: 0x4276, 0x2701: 0x427b, 0x2702: 0x427f, 0x2703: 0x4283, 0x2704: 0x4287, 0x2705: 0x428b, - 0x2706: 0x428f, 0x2707: 0x4293, 0x2708: 0x4298, 0x2709: 0x429d, 0x270a: 0x42a2, 0x270b: 0x3f14, - 0x270c: 0x42a7, 0x270d: 0x42ab, 0x270e: 0x42af, 0x270f: 0x42b3, 0x2710: 0x42b7, 0x2711: 0x42bb, - 0x2712: 0x42bf, 0x2713: 0x42c3, 0x2714: 0x42c7, 0x2715: 0x42cb, 0x2716: 0x42cf, 0x2717: 0x42d3, - 0x2718: 0x2c31, 0x2719: 0x42d8, 0x271a: 0x42dc, 0x271b: 0x42e0, 0x271c: 0x42e4, 0x271d: 0x42e8, - 0x271e: 0x42ec, 0x271f: 0x2f5d, 0x2720: 0x42f0, 0x2721: 0x42f4, 0x2722: 0x42f8, 0x2723: 0x42fc, - 0x2724: 0x4300, 0x2725: 0x4305, 0x2726: 0x430a, 0x2727: 0x430f, 0x2728: 0x4313, 0x2729: 0x4317, - 0x272a: 0x431b, 0x272b: 0x431f, 0x272c: 0x4324, 0x272d: 0x4328, 0x272e: 0x432d, 0x272f: 0x4331, - 0x2730: 0x4335, 0x2731: 0x433a, 0x2732: 0x433f, 0x2733: 0x4343, 0x2734: 0x2b45, 0x2735: 0x4347, - 0x2736: 0x434b, 0x2737: 0x434f, 0x2738: 0x4353, 0x2739: 0x4357, 0x273a: 0x435b, 0x273b: 0x306a, - 0x273c: 0x435f, 0x273d: 0x4363, 0x273e: 0x4367, 0x273f: 0x436b, - // Block 0x9d, offset 0x2740 - 0x2740: 0x436f, 0x2741: 0x4373, 0x2742: 0x4377, 0x2743: 0x437b, 0x2744: 0x1a66, 0x2745: 0x437f, - 0x2746: 0x4384, 0x2747: 0x4388, 0x2748: 0x438c, 0x2749: 0x4390, 0x274a: 0x4394, 0x274b: 0x4398, - 0x274c: 0x439d, 0x274d: 0x43a2, 0x274e: 0x43a6, 0x274f: 0x43aa, 0x2750: 0x307e, 0x2751: 0x3082, - 0x2752: 0x1a82, 0x2753: 0x43ae, 0x2754: 0x43b3, 0x2755: 0x43b7, 0x2756: 0x43bb, 0x2757: 0x43bf, - 0x2758: 0x43c3, 0x2759: 0x43c8, 0x275a: 0x43cd, 0x275b: 0x43d1, 0x275c: 0x43d5, 0x275d: 0x43d9, - 0x275e: 0x43de, 0x275f: 0x3086, 0x2760: 0x43e2, 0x2761: 0x43e7, 0x2762: 0x43ec, 0x2763: 0x43f0, - 0x2764: 0x43f4, 0x2765: 0x43f8, 0x2766: 0x43fd, 0x2767: 0x4401, 0x2768: 0x4405, 0x2769: 0x4409, - 0x276a: 0x440d, 0x276b: 0x4411, 0x276c: 0x4415, 0x276d: 0x4419, 0x276e: 0x441e, 0x276f: 0x4422, - 0x2770: 0x4426, 0x2771: 0x442a, 0x2772: 0x442f, 0x2773: 0x4433, 0x2774: 0x4437, 0x2775: 0x443b, - 0x2776: 0x443f, 0x2777: 0x4444, 0x2778: 0x4449, 0x2779: 0x444d, 0x277a: 0x4451, 0x277b: 0x4455, - 0x277c: 0x445a, 0x277d: 0x445e, 0x277e: 0x309e, 0x277f: 0x309e, - // Block 0x9e, offset 0x2780 - 0x2780: 0x4463, 0x2781: 0x4467, 0x2782: 0x446c, 0x2783: 0x4470, 0x2784: 0x4474, 0x2785: 0x4478, - 0x2786: 0x447c, 0x2787: 0x4480, 0x2788: 0x4484, 0x2789: 0x4488, 0x278a: 0x30a2, 0x278b: 0x448d, - 0x278c: 0x4491, 0x278d: 0x4495, 0x278e: 0x4499, 0x278f: 0x449d, 0x2790: 0x44a1, 0x2791: 0x44a6, - 0x2792: 0x44aa, 0x2793: 0x44af, 0x2794: 0x44b4, 0x2795: 0x1b42, 0x2796: 0x44b9, 0x2797: 0x1b52, - 0x2798: 0x44bd, 0x2799: 0x44c1, 0x279a: 0x44c5, 0x279b: 0x44c9, 0x279c: 0x1b66, 0x279d: 0x44cd, + 0x1040: 0x436f, 0x1041: 0x4373, 0x1042: 0x4377, 0x1043: 0x437b, 0x1044: 0x1a66, 0x1045: 0x437f, + 0x1046: 0x4384, 0x1047: 0x4388, 0x1048: 0x438c, 0x1049: 0x4390, 0x104a: 0x4394, 0x104b: 0x4398, + 0x104c: 0x439d, 0x104d: 0x43a2, 0x104e: 0x43a6, 0x104f: 0x43aa, 0x1050: 0x307e, 0x1051: 0x3082, + 0x1052: 0x1a82, 0x1053: 0x43ae, 0x1054: 0x43b3, 0x1055: 0x43b7, 0x1056: 0x43bb, 0x1057: 0x43bf, + 0x1058: 0x43c3, 0x1059: 0x43c8, 0x105a: 0x43cd, 0x105b: 0x43d1, 0x105c: 0x43d5, 0x105d: 0x43d9, + 0x105e: 0x43de, 0x105f: 0x3086, 0x1060: 0x43e2, 0x1061: 0x43e7, 0x1062: 0x43ec, 0x1063: 0x43f0, + 0x1064: 0x43f4, 0x1065: 0x43f8, 0x1066: 0x43fd, 0x1067: 0x4401, 0x1068: 0x4405, 0x1069: 0x4409, + 0x106a: 0x440d, 0x106b: 0x4411, 0x106c: 0x4415, 0x106d: 0x4419, 0x106e: 0x441e, 0x106f: 0x4422, + 0x1070: 0x4426, 0x1071: 0x442a, 0x1072: 0x442f, 0x1073: 0x4433, 0x1074: 0x4437, 0x1075: 0x443b, + 0x1076: 0x443f, 0x1077: 0x4444, 0x1078: 0x4449, 0x1079: 0x444d, 0x107a: 0x4451, 0x107b: 0x4455, + 0x107c: 0x445a, 0x107d: 0x445e, 0x107e: 0x309e, 0x107f: 0x309e, +} + +// nfkcDecompSparseOffset: 93 entries, 186 bytes +var nfkcDecompSparseOffset = []uint16{0x0, 0xc, 0x16, 0x1e, 0x24, 0x27, 0x31, 0x37, 0x3e, 0x44, 0x4c, 0x59, 0x60, 0x66, 0x6e, 0x70, 0x72, 0x74, 0x78, 0x7c, 0x7e, 0x82, 0x85, 0x88, 0x8c, 0x8e, 0x90, 0x92, 0x96, 0x98, 0x9c, 0x9e, 0xa0, 0xa2, 0xa4, 0xae, 0xb6, 0xb8, 0xba, 0xc3, 0xc6, 0xcd, 0xd8, 0xe6, 0xf4, 0xfe, 0x102, 0x104, 0x10e, 0x11a, 0x11f, 0x122, 0x124, 0x126, 0x129, 0x12b, 0x12d, 0x12f, 0x131, 0x133, 0x135, 0x137, 0x139, 0x13b, 0x140, 0x14f, 0x15d, 0x15f, 0x161, 0x169, 0x179, 0x17b, 0x186, 0x18d, 0x198, 0x1a4, 0x1b5, 0x1c6, 0x1cd, 0x1de, 0x1ec, 0x1fa, 0x209, 0x21a, 0x21f, 0x22c, 0x230, 0x234, 0x238, 0x23a, 0x249, 0x24b, 0x24f} + +// nfkcDecompSparseValues: 605 entries, 2420 bytes +var nfkcDecompSparseValues = [605]valueRange{ + // Block 0x0, offset 0x1 + {value: 0x0002, lo: 0x0b}, + {value: 0x0001, lo: 0xa0, hi: 0xa0}, + {value: 0x0003, lo: 0xa8, hi: 0xa8}, + {value: 0x0007, lo: 0xaa, hi: 0xaa}, + {value: 0x0009, lo: 0xaf, hi: 0xaf}, + {value: 0x000d, lo: 0xb2, hi: 0xb4}, + {value: 0x0015, lo: 0xb5, hi: 0xb5}, + {value: 0x0018, lo: 0xb8, hi: 0xb8}, + {value: 0x001c, lo: 0xb9, hi: 0xba}, + {value: 0x0020, lo: 0xbc, hi: 0xbc}, + {value: 0x0026, lo: 0xbd, hi: 0xbd}, + {value: 0x002c, lo: 0xbe, hi: 0xbe}, + // Block 0x1, offset 0x2 + {value: 0x0004, lo: 0x09}, + {value: 0x0032, lo: 0x80, hi: 0x85}, + {value: 0x004a, lo: 0x87, hi: 0x8f}, + {value: 0x006e, lo: 0x91, hi: 0x96}, + {value: 0x0086, lo: 0x99, hi: 0x9d}, + {value: 0x009a, lo: 0xa0, hi: 0xa5}, + {value: 0x00b2, lo: 0xa7, hi: 0xaf}, + {value: 0x00d6, lo: 0xb1, hi: 0xb6}, + {value: 0x00ee, lo: 0xb9, hi: 0xbd}, + {value: 0x0102, lo: 0xbf, hi: 0xbf}, + // Block 0x2, offset 0x3 + {value: 0x0004, lo: 0x07}, + {value: 0x0106, lo: 0x80, hi: 0x8f}, + {value: 0x0146, lo: 0x92, hi: 0xa5}, + {value: 0x0196, lo: 0xa8, hi: 0xb0}, + {value: 0x01ba, lo: 0xb2, hi: 0xb2}, + {value: 0x01bd, lo: 0xb3, hi: 0xb3}, + {value: 0x01c0, lo: 0xb4, hi: 0xb7}, + {value: 0x01d0, lo: 0xb9, hi: 0xbf}, + // Block 0x3, offset 0x4 + {value: 0x0004, lo: 0x05}, + {value: 0x01ec, lo: 0x80, hi: 0x80}, + {value: 0x01f0, lo: 0x83, hi: 0x89}, + {value: 0x020c, lo: 0x8c, hi: 0x91}, + {value: 0x0224, lo: 0x94, hi: 0xa5}, + {value: 0x026c, lo: 0xa8, hi: 0xbf}, + // Block 0x4, offset 0x5 + {value: 0x0004, lo: 0x02}, + {value: 0x02ca, lo: 0xa0, hi: 0xa1}, + {value: 0x02d2, lo: 0xaf, hi: 0xb0}, + // Block 0x5, offset 0x6 + {value: 0x0004, lo: 0x09}, + {value: 0x03d8, lo: 0x80, hi: 0x9b}, + {value: 0x0448, lo: 0x9e, hi: 0x9f}, + {value: 0x0450, lo: 0xa6, hi: 0xaa}, + {value: 0x0466, lo: 0xab, hi: 0xab}, + {value: 0x046c, lo: 0xac, hi: 0xac}, + {value: 0x0472, lo: 0xad, hi: 0xad}, + {value: 0x0478, lo: 0xae, hi: 0xb0}, + {value: 0x0486, lo: 0xb1, hi: 0xb1}, + {value: 0x048c, lo: 0xb2, hi: 0xb3}, + // Block 0x6, offset 0x7 + {value: 0x0002, lo: 0x05}, + {value: 0x0494, lo: 0xb0, hi: 0xb1}, + {value: 0x0499, lo: 0xb2, hi: 0xb4}, + {value: 0x04a0, lo: 0xb5, hi: 0xb5}, + {value: 0x04a3, lo: 0xb6, hi: 0xb6}, + {value: 0x04a6, lo: 0xb7, hi: 0xb8}, + // Block 0x7, offset 0x8 + {value: 0x0004, lo: 0x06}, + {value: 0x04aa, lo: 0x98, hi: 0x9d}, + {value: 0x04c2, lo: 0xa0, hi: 0xa0}, + {value: 0x04c5, lo: 0xa1, hi: 0xa1}, + {value: 0x02c8, lo: 0xa2, hi: 0xa2}, + {value: 0x04c7, lo: 0xa3, hi: 0xa3}, + {value: 0x04c9, lo: 0xa4, hi: 0xa4}, + // Block 0x8, offset 0x9 + {value: 0x0003, lo: 0x05}, + {value: 0x04cc, lo: 0x80, hi: 0x81}, + {value: 0x04d2, lo: 0x83, hi: 0x84}, + {value: 0x04da, lo: 0xb4, hi: 0xb4}, + {value: 0x04dd, lo: 0xba, hi: 0xba}, + {value: 0x04e1, lo: 0xbe, hi: 0xbe}, + // Block 0x9, offset 0xa + {value: 0x0005, lo: 0x07}, + {value: 0x0011, lo: 0x84, hi: 0x84}, + {value: 0x04e8, lo: 0x85, hi: 0x85}, + {value: 0x04ee, lo: 0x86, hi: 0x87}, + {value: 0x04f6, lo: 0x88, hi: 0x8a}, + {value: 0x0505, lo: 0x8c, hi: 0x8c}, + {value: 0x050a, lo: 0x8e, hi: 0x90}, + {value: 0x051b, lo: 0xaa, hi: 0xb0}, + // Block 0xa, offset 0xb + {value: 0x0003, lo: 0x0c}, + {value: 0x0540, lo: 0x8a, hi: 0x8a}, + {value: 0x0545, lo: 0x8b, hi: 0x8b}, + {value: 0x054a, lo: 0x8c, hi: 0x8c}, + {value: 0x054f, lo: 0x8d, hi: 0x8d}, + {value: 0x0554, lo: 0x8e, hi: 0x8e}, + {value: 0x0559, lo: 0x90, hi: 0x92}, + {value: 0x050a, lo: 0x93, hi: 0x93}, + {value: 0x0520, lo: 0x94, hi: 0x94}, + {value: 0x056c, lo: 0x95, hi: 0x96}, + {value: 0x0572, lo: 0xb0, hi: 0xb2}, + {value: 0x057b, lo: 0xb4, hi: 0xb5}, + {value: 0x0581, lo: 0xb9, hi: 0xb9}, + // Block 0xb, offset 0xc + {value: 0x0005, lo: 0x06}, + {value: 0x0584, lo: 0x80, hi: 0x81}, + {value: 0x058e, lo: 0x83, hi: 0x83}, + {value: 0x0593, lo: 0x87, hi: 0x87}, + {value: 0x0598, lo: 0x8c, hi: 0x8e}, + {value: 0x05a7, lo: 0x99, hi: 0x99}, + {value: 0x05ac, lo: 0xb9, hi: 0xb9}, + // Block 0xc, offset 0xd + {value: 0x0005, lo: 0x05}, + {value: 0x05b1, lo: 0x90, hi: 0x91}, + {value: 0x05bb, lo: 0x93, hi: 0x93}, + {value: 0x05c0, lo: 0x97, hi: 0x97}, + {value: 0x05c5, lo: 0x9c, hi: 0x9e}, + {value: 0x05d4, lo: 0xb6, hi: 0xb7}, + // Block 0xd, offset 0xe + {value: 0x0005, lo: 0x07}, + {value: 0x05de, lo: 0x81, hi: 0x82}, + {value: 0x05e8, lo: 0x90, hi: 0x93}, + {value: 0x05fc, lo: 0x96, hi: 0x97}, + {value: 0x0606, lo: 0x9a, hi: 0x9f}, + {value: 0x0624, lo: 0xa2, hi: 0xa7}, + {value: 0x0642, lo: 0xaa, hi: 0xb5}, + {value: 0x067e, lo: 0xb8, hi: 0xb9}, + // Block 0xe, offset 0xf + {value: 0x0000, lo: 0x01}, + {value: 0x0688, lo: 0x87, hi: 0x87}, + // Block 0xf, offset 0x10 + {value: 0x0005, lo: 0x01}, + {value: 0x068d, lo: 0xa2, hi: 0xa6}, + // Block 0x10, offset 0x11 + {value: 0x0005, lo: 0x01}, + {value: 0x06a6, lo: 0xb5, hi: 0xb8}, + // Block 0x11, offset 0x12 + {value: 0x0005, lo: 0x03}, + {value: 0x06ba, lo: 0x80, hi: 0x80}, + {value: 0x06bf, lo: 0x82, hi: 0x82}, + {value: 0x06c4, lo: 0x93, hi: 0x93}, + // Block 0x12, offset 0x13 + {value: 0x0007, lo: 0x03}, + {value: 0x06c9, lo: 0xa9, hi: 0xa9}, + {value: 0x06d0, lo: 0xb1, hi: 0xb1}, + {value: 0x06d7, lo: 0xb4, hi: 0xb4}, + // Block 0x13, offset 0x14 + {value: 0x0007, lo: 0x01}, + {value: 0x06de, lo: 0x98, hi: 0x9f}, + // Block 0x14, offset 0x15 + {value: 0x0007, lo: 0x03}, + {value: 0x0716, lo: 0x8b, hi: 0x8c}, + {value: 0x0724, lo: 0x9c, hi: 0x9d}, + {value: 0x0732, lo: 0x9f, hi: 0x9f}, + // Block 0x15, offset 0x16 + {value: 0x0007, lo: 0x02}, + {value: 0x0739, lo: 0xb3, hi: 0xb3}, + {value: 0x0740, lo: 0xb6, hi: 0xb6}, + // Block 0x16, offset 0x17 + {value: 0x0007, lo: 0x02}, + {value: 0x0747, lo: 0x99, hi: 0x9b}, + {value: 0x075c, lo: 0x9e, hi: 0x9e}, + // Block 0x17, offset 0x18 + {value: 0x0007, lo: 0x03}, + {value: 0x0763, lo: 0x88, hi: 0x88}, + {value: 0x076a, lo: 0x8b, hi: 0x8c}, + {value: 0x0778, lo: 0x9c, hi: 0x9d}, + // Block 0x18, offset 0x19 + {value: 0x0000, lo: 0x01}, + {value: 0x0786, lo: 0x94, hi: 0x94}, + // Block 0x19, offset 0x1a + {value: 0x0007, lo: 0x01}, + {value: 0x078d, lo: 0x8a, hi: 0x8c}, + // Block 0x1a, offset 0x1b + {value: 0x0000, lo: 0x01}, + {value: 0x07a2, lo: 0x88, hi: 0x88}, + // Block 0x1b, offset 0x1c + {value: 0x0007, lo: 0x03}, + {value: 0x07a9, lo: 0x80, hi: 0x80}, + {value: 0x07b0, lo: 0x87, hi: 0x88}, + {value: 0x07be, lo: 0x8a, hi: 0x8b}, + // Block 0x1c, offset 0x1d + {value: 0x0007, lo: 0x01}, + {value: 0x07cf, lo: 0x8a, hi: 0x8c}, + // Block 0x1d, offset 0x1e + {value: 0x0007, lo: 0x03}, + {value: 0x07e4, lo: 0x9a, hi: 0x9a}, + {value: 0x07eb, lo: 0x9c, hi: 0x9d}, + {value: 0x07fc, lo: 0x9e, hi: 0x9e}, + // Block 0x1e, offset 0x1f + {value: 0x0000, lo: 0x01}, + {value: 0x0803, lo: 0xb3, hi: 0xb3}, + // Block 0x1f, offset 0x20 + {value: 0x0000, lo: 0x01}, + {value: 0x080a, lo: 0xb3, hi: 0xb3}, + // Block 0x20, offset 0x21 + {value: 0x0007, lo: 0x01}, + {value: 0x0811, lo: 0x9c, hi: 0x9d}, + // Block 0x21, offset 0x22 + {value: 0x0000, lo: 0x01}, + {value: 0x081f, lo: 0x8c, hi: 0x8c}, + // Block 0x22, offset 0x23 + {value: 0x0007, lo: 0x09}, + {value: 0x0823, lo: 0x83, hi: 0x83}, + {value: 0x082a, lo: 0x8d, hi: 0x8d}, + {value: 0x0831, lo: 0x92, hi: 0x92}, + {value: 0x0838, lo: 0x97, hi: 0x97}, + {value: 0x083f, lo: 0x9c, hi: 0x9c}, + {value: 0x0846, lo: 0xa9, hi: 0xa9}, + {value: 0x084d, lo: 0xb3, hi: 0xb3}, + {value: 0x0854, lo: 0xb5, hi: 0xb7}, + {value: 0x086c, lo: 0xb8, hi: 0xb9}, + // Block 0x23, offset 0x24 + {value: 0x0007, lo: 0x07}, + {value: 0x087d, lo: 0x81, hi: 0x81}, + {value: 0x0884, lo: 0x93, hi: 0x93}, + {value: 0x088b, lo: 0x9d, hi: 0x9d}, + {value: 0x0892, lo: 0xa2, hi: 0xa2}, + {value: 0x0899, lo: 0xa7, hi: 0xa7}, + {value: 0x08a0, lo: 0xac, hi: 0xac}, + {value: 0x08a7, lo: 0xb9, hi: 0xb9}, + // Block 0x24, offset 0x25 + {value: 0x0000, lo: 0x01}, + {value: 0x08ae, lo: 0xa6, hi: 0xa6}, + // Block 0x25, offset 0x26 + {value: 0x0000, lo: 0x01}, + {value: 0x08b5, lo: 0xbc, hi: 0xbc}, + // Block 0x26, offset 0x27 + {value: 0x0007, lo: 0x08}, + {value: 0x08b9, lo: 0x86, hi: 0x86}, + {value: 0x08c0, lo: 0x88, hi: 0x88}, + {value: 0x08c7, lo: 0x8a, hi: 0x8a}, + {value: 0x08ce, lo: 0x8c, hi: 0x8c}, + {value: 0x08d5, lo: 0x8e, hi: 0x8e}, + {value: 0x08dc, lo: 0x92, hi: 0x92}, + {value: 0x08e3, lo: 0xbb, hi: 0xbb}, + {value: 0x08ea, lo: 0xbd, hi: 0xbd}, + // Block 0x27, offset 0x28 + {value: 0x0007, lo: 0x02}, + {value: 0x08f1, lo: 0x80, hi: 0x81}, + {value: 0x08ff, lo: 0x83, hi: 0x83}, + // Block 0x28, offset 0x29 + {value: 0x0002, lo: 0x06}, + {value: 0x0906, lo: 0xac, hi: 0xad}, + {value: 0x090b, lo: 0xae, hi: 0xae}, + {value: 0x090d, lo: 0xb0, hi: 0xb2}, + {value: 0x0914, lo: 0xb3, hi: 0xba}, + {value: 0x0924, lo: 0xbc, hi: 0xbd}, + {value: 0x0929, lo: 0xbe, hi: 0xbf}, + // Block 0x29, offset 0x2a + {value: 0x0003, lo: 0x0a}, + {value: 0x0981, lo: 0x9b, hi: 0x9c}, + {value: 0x0986, lo: 0x9d, hi: 0x9e}, + {value: 0x0949, lo: 0x9f, hi: 0x9f}, + {value: 0x098c, lo: 0xa0, hi: 0xa0}, + {value: 0x098e, lo: 0xa1, hi: 0xa7}, + {value: 0x09a4, lo: 0xa8, hi: 0xaa}, + {value: 0x09ae, lo: 0xab, hi: 0xb8}, + {value: 0x09d9, lo: 0xb9, hi: 0xbb}, + {value: 0x09e1, lo: 0xbc, hi: 0xbe}, + {value: 0x055c, lo: 0xbf, hi: 0xbf}, + // Block 0x2a, offset 0x2b + {value: 0x0004, lo: 0x0d}, + {value: 0x09ea, lo: 0x80, hi: 0x88}, + {value: 0x0a10, lo: 0x89, hi: 0x89}, + {value: 0x0a16, lo: 0x8a, hi: 0x94}, + {value: 0x0a44, lo: 0x95, hi: 0x95}, + {value: 0x0a4a, lo: 0x96, hi: 0x96}, + {value: 0x0a50, lo: 0x97, hi: 0x97}, + {value: 0x0a56, lo: 0x98, hi: 0x9c}, + {value: 0x0a6c, lo: 0x9d, hi: 0x9d}, + {value: 0x0a72, lo: 0x9e, hi: 0xae}, + {value: 0x0ab8, lo: 0xaf, hi: 0xaf}, + {value: 0x0abe, lo: 0xb0, hi: 0xb8}, + {value: 0x0ae4, lo: 0xb9, hi: 0xb9}, + {value: 0x0aea, lo: 0xba, hi: 0xbf}, + // Block 0x2b, offset 0x2c + {value: 0x0000, lo: 0x0d}, + {value: 0x0001, lo: 0x80, hi: 0x8a}, + {value: 0x1436, lo: 0x91, hi: 0x91}, + {value: 0x143a, lo: 0x97, hi: 0x97}, + {value: 0x143e, lo: 0xa4, hi: 0xa4}, + {value: 0x1440, lo: 0xa5, hi: 0xa5}, + {value: 0x1443, lo: 0xa6, hi: 0xa6}, + {value: 0x0001, lo: 0xaf, hi: 0xaf}, + {value: 0x1447, lo: 0xb3, hi: 0xb3}, + {value: 0x144e, lo: 0xb4, hi: 0xb4}, + {value: 0x1458, lo: 0xb6, hi: 0xb6}, + {value: 0x145f, lo: 0xb7, hi: 0xb7}, + {value: 0x1469, lo: 0xbc, hi: 0xbc}, + {value: 0x146c, lo: 0xbe, hi: 0xbe}, + // Block 0x2c, offset 0x2d + {value: 0x0002, lo: 0x09}, + {value: 0x1470, lo: 0x87, hi: 0x87}, + {value: 0x1473, lo: 0x88, hi: 0x88}, + {value: 0x1476, lo: 0x89, hi: 0x89}, + {value: 0x1479, lo: 0x97, hi: 0x97}, + {value: 0x0001, lo: 0x9f, hi: 0x9f}, + {value: 0x1486, lo: 0xb0, hi: 0xb0}, + {value: 0x097c, lo: 0xb1, hi: 0xb1}, + {value: 0x1488, lo: 0xb4, hi: 0xbb}, + {value: 0x149a, lo: 0xbc, hi: 0xbf}, + // Block 0x2d, offset 0x2e + {value: 0x0006, lo: 0x03}, + {value: 0x1599, lo: 0x89, hi: 0x89}, + {value: 0x159f, lo: 0x9a, hi: 0x9b}, + {value: 0x15ab, lo: 0xae, hi: 0xae}, + // Block 0x2e, offset 0x2f + {value: 0x0006, lo: 0x01}, + {value: 0x15b1, lo: 0x8d, hi: 0x8f}, + // Block 0x2f, offset 0x30 + {value: 0x0006, lo: 0x09}, + {value: 0x15c3, lo: 0x84, hi: 0x84}, + {value: 0x15c9, lo: 0x89, hi: 0x89}, + {value: 0x15cf, lo: 0x8c, hi: 0x8c}, + {value: 0x15d5, lo: 0xa4, hi: 0xa4}, + {value: 0x15db, lo: 0xa6, hi: 0xa6}, + {value: 0x15e1, lo: 0xac, hi: 0xac}, + {value: 0x15e8, lo: 0xad, hi: 0xad}, + {value: 0x15f2, lo: 0xaf, hi: 0xaf}, + {value: 0x15f9, lo: 0xb0, hi: 0xb0}, + // Block 0x30, offset 0x31 + {value: 0x0006, lo: 0x0b}, + {value: 0x1603, lo: 0x81, hi: 0x81}, + {value: 0x1609, lo: 0x84, hi: 0x84}, + {value: 0x160f, lo: 0x87, hi: 0x87}, + {value: 0x1615, lo: 0x89, hi: 0x89}, + {value: 0x161b, lo: 0xa0, hi: 0xa0}, + {value: 0x161f, lo: 0xa2, hi: 0xa2}, + {value: 0x1625, lo: 0xad, hi: 0xae}, + {value: 0x162f, lo: 0xaf, hi: 0xaf}, + {value: 0x1633, lo: 0xb0, hi: 0xb1}, + {value: 0x163f, lo: 0xb4, hi: 0xb5}, + {value: 0x164b, lo: 0xb8, hi: 0xb9}, + // Block 0x31, offset 0x32 + {value: 0x0006, lo: 0x04}, + {value: 0x1657, lo: 0x80, hi: 0x81}, + {value: 0x1663, lo: 0x84, hi: 0x85}, + {value: 0x166f, lo: 0x88, hi: 0x89}, + {value: 0x167b, lo: 0xac, hi: 0xaf}, + // Block 0x32, offset 0x33 + {value: 0x0006, lo: 0x02}, + {value: 0x1693, lo: 0xa0, hi: 0xa3}, + {value: 0x16ab, lo: 0xaa, hi: 0xad}, + // Block 0x33, offset 0x34 + {value: 0x0004, lo: 0x01}, + {value: 0x16c3, lo: 0xa9, hi: 0xaa}, + // Block 0x34, offset 0x35 + {value: 0x0000, lo: 0x01}, + {value: 0x17fc, lo: 0x8c, hi: 0x8c}, + // Block 0x35, offset 0x36 + {value: 0x0004, lo: 0x02}, + {value: 0x1809, lo: 0xb4, hi: 0xb5}, + {value: 0x1810, lo: 0xb6, hi: 0xb6}, + // Block 0x36, offset 0x37 + {value: 0x0000, lo: 0x01}, + {value: 0x1814, lo: 0x9c, hi: 0x9c}, + // Block 0x37, offset 0x38 + {value: 0x10c6, lo: 0x01}, + {value: 0x0499, lo: 0xbc, hi: 0xbd}, + // Block 0x38, offset 0x39 + {value: 0x0000, lo: 0x01}, + {value: 0x181a, lo: 0xaf, hi: 0xaf}, + // Block 0x39, offset 0x3a + {value: 0x0000, lo: 0x01}, + {value: 0x181e, lo: 0x9f, hi: 0x9f}, + // Block 0x3a, offset 0x3b + {value: 0x0000, lo: 0x01}, + {value: 0x1822, lo: 0xb3, hi: 0xb3}, + // Block 0x3b, offset 0x3c + {value: 0x0004, lo: 0x01}, + {value: 0x1826, lo: 0x80, hi: 0xbf}, + // Block 0x3c, offset 0x3d + {value: 0x0004, lo: 0x01}, + {value: 0x1926, lo: 0x80, hi: 0xbf}, + // Block 0x3d, offset 0x3e + {value: 0x0004, lo: 0x01}, + {value: 0x1a26, lo: 0x80, hi: 0xbf}, + // Block 0x3e, offset 0x3f + {value: 0x0004, lo: 0x01}, + {value: 0x1b26, lo: 0x80, hi: 0x95}, + // Block 0x3f, offset 0x40 + {value: 0x0300, lo: 0x04}, + {value: 0x0001, lo: 0x80, hi: 0x80}, + {value: 0x1b7e, lo: 0xb6, hi: 0xb6}, + {value: 0x1882, lo: 0xb8, hi: 0xb9}, + {value: 0x1b86, lo: 0xba, hi: 0xba}, + // Block 0x40, offset 0x41 + {value: 0x0007, lo: 0x0e}, + {value: 0x1c39, lo: 0x94, hi: 0x94}, + {value: 0x1c40, lo: 0x9b, hi: 0x9b}, + {value: 0x1c45, lo: 0x9c, hi: 0x9c}, + {value: 0x1c4a, lo: 0x9e, hi: 0x9f}, + {value: 0x1c58, lo: 0xac, hi: 0xac}, + {value: 0x1c5f, lo: 0xae, hi: 0xae}, + {value: 0x1c66, lo: 0xb0, hi: 0xb0}, + {value: 0x1c6d, lo: 0xb2, hi: 0xb2}, + {value: 0x1c74, lo: 0xb4, hi: 0xb4}, + {value: 0x1c7b, lo: 0xb6, hi: 0xb6}, + {value: 0x1c82, lo: 0xb8, hi: 0xb8}, + {value: 0x1c89, lo: 0xba, hi: 0xba}, + {value: 0x1c90, lo: 0xbc, hi: 0xbc}, + {value: 0x1c97, lo: 0xbe, hi: 0xbe}, + // Block 0x41, offset 0x42 + {value: 0x0007, lo: 0x0d}, + {value: 0x1c9e, lo: 0x80, hi: 0x80}, + {value: 0x1ca5, lo: 0x82, hi: 0x82}, + {value: 0x1cac, lo: 0x85, hi: 0x85}, + {value: 0x1cb3, lo: 0x87, hi: 0x87}, + {value: 0x1cba, lo: 0x89, hi: 0x89}, + {value: 0x1cc1, lo: 0x90, hi: 0x91}, + {value: 0x1ccf, lo: 0x93, hi: 0x94}, + {value: 0x1cdd, lo: 0x96, hi: 0x97}, + {value: 0x1ceb, lo: 0x99, hi: 0x9a}, + {value: 0x1cf9, lo: 0x9c, hi: 0x9d}, + {value: 0x1d07, lo: 0xb4, hi: 0xb4}, + {value: 0x1d0e, lo: 0xb7, hi: 0xba}, + {value: 0x1d2a, lo: 0xbe, hi: 0xbf}, + // Block 0x42, offset 0x43 + {value: 0x0004, lo: 0x01}, + {value: 0x1d38, lo: 0xb1, hi: 0xbf}, + // Block 0x43, offset 0x44 + {value: 0x0004, lo: 0x01}, + {value: 0x1d74, lo: 0x80, hi: 0xbf}, + // Block 0x44, offset 0x45 + {value: 0x0004, lo: 0x07}, + {value: 0x1e74, lo: 0x80, hi: 0x8e}, + {value: 0x1826, lo: 0x92, hi: 0x92}, + {value: 0x183e, lo: 0x93, hi: 0x93}, + {value: 0x1eb0, lo: 0x94, hi: 0x99}, + {value: 0x1836, lo: 0x9a, hi: 0x9a}, + {value: 0x1ec8, lo: 0x9b, hi: 0x9e}, + {value: 0x1846, lo: 0x9f, hi: 0x9f}, + // Block 0x45, offset 0x46 + {value: 0x0004, lo: 0x0f}, + {value: 0x221c, lo: 0x80, hi: 0x80}, + {value: 0x2221, lo: 0x81, hi: 0x81}, + {value: 0x2226, lo: 0x82, hi: 0x82}, + {value: 0x222b, lo: 0x83, hi: 0x83}, + {value: 0x2230, lo: 0x84, hi: 0x84}, + {value: 0x2235, lo: 0x85, hi: 0x85}, + {value: 0x223a, lo: 0x86, hi: 0x86}, + {value: 0x223f, lo: 0x87, hi: 0x87}, + {value: 0x2244, lo: 0x88, hi: 0x88}, + {value: 0x2249, lo: 0x89, hi: 0x89}, + {value: 0x224f, lo: 0x8a, hi: 0x8a}, + {value: 0x2255, lo: 0x8b, hi: 0x8b}, + {value: 0x225b, lo: 0x8c, hi: 0x8c}, + {value: 0x225e, lo: 0x8d, hi: 0x8e}, + {value: 0x2265, lo: 0x8f, hi: 0xbe}, + // Block 0x46, offset 0x47 + {value: 0x0000, lo: 0x01}, + {value: 0x2a7d, lo: 0xb0, hi: 0xb0}, + // Block 0x47, offset 0x48 + {value: 0x0004, lo: 0x0a}, + {value: 0x2a81, lo: 0x80, hi: 0x81}, + {value: 0x1a9e, lo: 0x82, hi: 0x82}, + {value: 0x2a89, lo: 0x83, hi: 0x86}, + {value: 0x1b76, lo: 0x87, hi: 0x87}, + {value: 0x1b76, lo: 0x88, hi: 0x88}, + {value: 0x2a99, lo: 0x89, hi: 0x89}, + {value: 0x1abe, lo: 0x8a, hi: 0x8a}, + {value: 0x2a9d, lo: 0x8b, hi: 0xb3}, + {value: 0x1a16, lo: 0xb4, hi: 0xb4}, + {value: 0x2b41, lo: 0xb5, hi: 0xbf}, + // Block 0x48, offset 0x49 + {value: 0x0004, lo: 0x06}, + {value: 0x1b3a, lo: 0x80, hi: 0x80}, + {value: 0x2b6d, lo: 0x81, hi: 0x9b}, + {value: 0x2ac1, lo: 0x9c, hi: 0x9c}, + {value: 0x2bd9, lo: 0x9d, hi: 0xb0}, + {value: 0x1aa6, lo: 0xb1, hi: 0xb1}, + {value: 0x2c29, lo: 0xb2, hi: 0xbf}, + // Block 0x49, offset 0x4a + {value: 0x0004, lo: 0x0a}, + {value: 0x2c61, lo: 0x80, hi: 0x80}, + {value: 0x18ba, lo: 0x81, hi: 0x81}, + {value: 0x2c65, lo: 0x82, hi: 0x89}, + {value: 0x186e, lo: 0x8a, hi: 0x8a}, + {value: 0x2c85, lo: 0x8b, hi: 0xa0}, + {value: 0x2c21, lo: 0xa1, hi: 0xa1}, + {value: 0x2cdd, lo: 0xa2, hi: 0xa9}, + {value: 0x2be1, lo: 0xaa, hi: 0xaa}, + {value: 0x2cfd, lo: 0xab, hi: 0xbe}, + {value: 0x2ac1, lo: 0xbf, hi: 0xbf}, + // Block 0x4a, offset 0x4b + {value: 0x0004, lo: 0x0b}, + {value: 0x2d4d, lo: 0x80, hi: 0x83}, + {value: 0x1b72, lo: 0x84, hi: 0x84}, + {value: 0x2d5d, lo: 0x85, hi: 0x90}, + {value: 0x2173, lo: 0x91, hi: 0x91}, + {value: 0x2d8d, lo: 0x92, hi: 0x9a}, + {value: 0x2be9, lo: 0x9b, hi: 0x9b}, + {value: 0x2db1, lo: 0x9c, hi: 0xa8}, + {value: 0x1aba, lo: 0xa9, hi: 0xa9}, + {value: 0x2de5, lo: 0xaa, hi: 0xb6}, + {value: 0x19f6, lo: 0xb7, hi: 0xb7}, + {value: 0x2e19, lo: 0xb8, hi: 0xbf}, + // Block 0x4b, offset 0x4c + {value: 0x0004, lo: 0x10}, + {value: 0x2e39, lo: 0x80, hi: 0x87}, + {value: 0x1a62, lo: 0x88, hi: 0x88}, + {value: 0x2e59, lo: 0x89, hi: 0x89}, + {value: 0x1a6e, lo: 0x8a, hi: 0x8a}, + {value: 0x2e5d, lo: 0x8b, hi: 0x8d}, + {value: 0x2e69, lo: 0x90, hi: 0x90}, + {value: 0x2e6d, lo: 0x92, hi: 0x92}, + {value: 0x2e71, lo: 0x95, hi: 0x9d}, + {value: 0x1a12, lo: 0x9e, hi: 0x9e}, + {value: 0x2e95, lo: 0xa0, hi: 0xa0}, + {value: 0x2e99, lo: 0xa2, hi: 0xa2}, + {value: 0x2e9d, lo: 0xa5, hi: 0xa6}, + {value: 0x2ea5, lo: 0xaa, hi: 0xad}, + {value: 0x2eb5, lo: 0xb0, hi: 0xbb}, + {value: 0x18d6, lo: 0xbc, hi: 0xbc}, + {value: 0x2ee5, lo: 0xbd, hi: 0xbf}, + // Block 0x4c, offset 0x4d + {value: 0x0004, lo: 0x10}, + {value: 0x2ef1, lo: 0x80, hi: 0x8b}, + {value: 0x2187, lo: 0x8c, hi: 0x8c}, + {value: 0x2f21, lo: 0x8d, hi: 0x90}, + {value: 0x2197, lo: 0x91, hi: 0x91}, + {value: 0x2f31, lo: 0x92, hi: 0x96}, + {value: 0x2cb1, lo: 0x97, hi: 0x97}, + {value: 0x2f45, lo: 0x98, hi: 0x9d}, + {value: 0x2f59, lo: 0x9e, hi: 0xa6}, + {value: 0x2e9d, lo: 0xa7, hi: 0xa7}, + {value: 0x2f7d, lo: 0xa8, hi: 0xac}, + {value: 0x2f92, lo: 0xad, hi: 0xad}, + {value: 0x2f96, lo: 0xb0, hi: 0xb7}, + {value: 0x2ecd, lo: 0xb8, hi: 0xb8}, + {value: 0x2fb6, lo: 0xb9, hi: 0xbb}, + {value: 0x2e69, lo: 0xbc, hi: 0xbc}, + {value: 0x2fc2, lo: 0xbd, hi: 0xbf}, + // Block 0x4d, offset 0x4e + {value: 0x0005, lo: 0x06}, + {value: 0x3277, lo: 0x80, hi: 0x80}, + {value: 0x327e, lo: 0x81, hi: 0x81}, + {value: 0x3285, lo: 0x82, hi: 0x82}, + {value: 0x326d, lo: 0x83, hi: 0x83}, + {value: 0x328c, lo: 0x84, hi: 0x84}, + {value: 0x3293, lo: 0x85, hi: 0xbf}, + // Block 0x4e, offset 0x4f + {value: 0x0005, lo: 0x10}, + {value: 0x356a, lo: 0x80, hi: 0x8b}, + {value: 0x3514, lo: 0x8c, hi: 0x8c}, + {value: 0x35a6, lo: 0x8d, hi: 0x90}, + {value: 0x3533, lo: 0x91, hi: 0xa7}, + {value: 0x3514, lo: 0xa8, hi: 0xa8}, + {value: 0x35a6, lo: 0xa9, hi: 0xac}, + {value: 0x3597, lo: 0xad, hi: 0xaf}, + {value: 0x3514, lo: 0xb0, hi: 0xb0}, + {value: 0x350f, lo: 0xb1, hi: 0xb1}, + {value: 0x3519, lo: 0xb2, hi: 0xb2}, + {value: 0x333d, lo: 0xb3, hi: 0xb3}, + {value: 0x3306, lo: 0xb4, hi: 0xb6}, + {value: 0x3597, lo: 0xb7, hi: 0xb9}, + {value: 0x333d, lo: 0xba, hi: 0xbb}, + {value: 0x35ba, lo: 0xbc, hi: 0xbc}, + {value: 0x35ba, lo: 0xbd, hi: 0xbd}, + // Block 0x4f, offset 0x50 + {value: 0x0007, lo: 0x0d}, + {value: 0x35bf, lo: 0x90, hi: 0x91}, + {value: 0x35c6, lo: 0x92, hi: 0x98}, + {value: 0x35f0, lo: 0x99, hi: 0x9f}, + {value: 0x361a, lo: 0xa0, hi: 0xa2}, + {value: 0x3628, lo: 0xa3, hi: 0xa4}, + {value: 0x362f, lo: 0xa5, hi: 0xa7}, + {value: 0x363d, lo: 0xa8, hi: 0xaa}, + {value: 0x364b, lo: 0xab, hi: 0xac}, + {value: 0x3652, lo: 0xad, hi: 0xaf}, + {value: 0x3660, lo: 0xb0, hi: 0xb1}, + {value: 0x3667, lo: 0xb2, hi: 0xb6}, + {value: 0x3683, lo: 0xb7, hi: 0xbc}, + {value: 0x36a6, lo: 0xbd, hi: 0xbf}, + // Block 0x50, offset 0x51 + {value: 0x0007, lo: 0x0d}, + {value: 0x36bb, lo: 0x80, hi: 0x83}, + {value: 0x36d0, lo: 0x84, hi: 0x85}, + {value: 0x36d7, lo: 0x86, hi: 0x87}, + {value: 0x36de, lo: 0x88, hi: 0x8f}, + {value: 0x3716, lo: 0x92, hi: 0x97}, + {value: 0x3739, lo: 0x98, hi: 0x9c}, + {value: 0x3755, lo: 0x9d, hi: 0xb3}, + {value: 0x36ad, lo: 0xb4, hi: 0xb4}, + {value: 0x36bb, lo: 0xb5, hi: 0xb5}, + {value: 0x37f6, lo: 0xb6, hi: 0xbb}, + {value: 0x3812, lo: 0xbc, hi: 0xbc}, + {value: 0x3804, lo: 0xbd, hi: 0xbd}, + {value: 0x3820, lo: 0xbe, hi: 0xbf}, + // Block 0x51, offset 0x52 + {value: 0x0009, lo: 0x0e}, + {value: 0x382e, lo: 0x80, hi: 0x80}, + {value: 0x3835, lo: 0x81, hi: 0x81}, + {value: 0x383c, lo: 0x82, hi: 0x82}, + {value: 0x3819, lo: 0x83, hi: 0x83}, + {value: 0x367c, lo: 0x84, hi: 0x84}, + {value: 0x3636, lo: 0x85, hi: 0x85}, + {value: 0x3843, lo: 0x86, hi: 0x86}, + {value: 0x384a, lo: 0x87, hi: 0x87}, + {value: 0x3851, lo: 0xb0, hi: 0xb0}, + {value: 0x3858, lo: 0xb1, hi: 0xb1}, + {value: 0x385f, lo: 0xb2, hi: 0xb9}, + {value: 0x38a5, lo: 0xba, hi: 0xba}, + {value: 0x38c7, lo: 0xbb, hi: 0xbb}, + {value: 0x38d7, lo: 0xbc, hi: 0xbc}, + // Block 0x52, offset 0x53 + {value: 0x0004, lo: 0x10}, + {value: 0x38e0, lo: 0x90, hi: 0x90}, + {value: 0x38e2, lo: 0x91, hi: 0x93}, + {value: 0x04e1, lo: 0x94, hi: 0x94}, + {value: 0x38ec, lo: 0x95, hi: 0x95}, + {value: 0x38ee, lo: 0x96, hi: 0x96}, + {value: 0x38f0, lo: 0x97, hi: 0x98}, + {value: 0x1443, lo: 0x99, hi: 0x99}, + {value: 0x1440, lo: 0xb0, hi: 0xb0}, + {value: 0x38f8, lo: 0xb1, hi: 0xb3}, + {value: 0x3900, lo: 0xb4, hi: 0xb4}, + {value: 0x149c, lo: 0xb5, hi: 0xb5}, + {value: 0x149e, lo: 0xb6, hi: 0xb6}, + {value: 0x3902, lo: 0xb7, hi: 0xb7}, + {value: 0x3904, lo: 0xb8, hi: 0xb8}, + {value: 0x3906, lo: 0xb9, hi: 0xbe}, + {value: 0x16c3, lo: 0xbf, hi: 0xbf}, + // Block 0x53, offset 0x54 + {value: 0x0004, lo: 0x04}, + {value: 0x22a5, lo: 0x80, hi: 0x9c}, + {value: 0x3a38, lo: 0x9d, hi: 0x9f}, + {value: 0x1e04, lo: 0xa0, hi: 0xa0}, + {value: 0x1d38, lo: 0xa1, hi: 0xbe}, + // Block 0x54, offset 0x55 + {value: 0x0004, lo: 0x0c}, + {value: 0x1db0, lo: 0x82, hi: 0x87}, + {value: 0x1dc8, lo: 0x8a, hi: 0x8f}, + {value: 0x1de0, lo: 0x92, hi: 0x97}, + {value: 0x1df8, lo: 0x9a, hi: 0x9c}, + {value: 0x3a44, lo: 0xa0, hi: 0xa0}, + {value: 0x3a47, lo: 0xa1, hi: 0xa1}, + {value: 0x3a4a, lo: 0xa2, hi: 0xa2}, + {value: 0x0009, lo: 0xa3, hi: 0xa3}, + {value: 0x3a4d, lo: 0xa4, hi: 0xa4}, + {value: 0x3a50, lo: 0xa5, hi: 0xa5}, + {value: 0x3a53, lo: 0xa6, hi: 0xa6}, + {value: 0x3a57, lo: 0xa8, hi: 0xae}, + // Block 0x55, offset 0x56 + {value: 0x0009, lo: 0x03}, + {value: 0x3a73, lo: 0x9a, hi: 0x9a}, + {value: 0x3a7c, lo: 0x9c, hi: 0x9c}, + {value: 0x3a85, lo: 0xab, hi: 0xab}, + // Block 0x56, offset 0x57 + {value: 0x000d, lo: 0x03}, + {value: 0x3a8e, lo: 0x9e, hi: 0x9e}, + {value: 0x3a97, lo: 0x9f, hi: 0x9f}, + {value: 0x3aa0, lo: 0xa0, hi: 0xa4}, + // Block 0x57, offset 0x58 + {value: 0x0009, lo: 0x03}, + {value: 0x3ae1, lo: 0xbb, hi: 0xbd}, + {value: 0x3b00, lo: 0xbe, hi: 0xbe}, + {value: 0x3b0d, lo: 0xbf, hi: 0xbf}, + // Block 0x58, offset 0x59 + {value: 0x0000, lo: 0x01}, + {value: 0x3b1a, lo: 0x80, hi: 0x80}, + // Block 0x59, offset 0x5a + {value: 0x0003, lo: 0x0e}, + {value: 0x14c8, lo: 0x80, hi: 0x80}, + {value: 0x092b, lo: 0x81, hi: 0x81}, + {value: 0x17f6, lo: 0x82, hi: 0x82}, + {value: 0x092d, lo: 0x83, hi: 0x83}, + {value: 0x092f, lo: 0x84, hi: 0x84}, + {value: 0x155f, lo: 0x85, hi: 0x85}, + {value: 0x0931, lo: 0x86, hi: 0x86}, + {value: 0x1570, lo: 0x87, hi: 0x87}, + {value: 0x17f8, lo: 0x88, hi: 0x88}, + {value: 0x14d4, lo: 0x89, hi: 0x89}, + {value: 0x3c2c, lo: 0x8a, hi: 0x8a}, + {value: 0x293d, lo: 0x8b, hi: 0x8b}, + {value: 0x3c2f, lo: 0x8c, hi: 0x8e}, + {value: 0x3c39, lo: 0x8f, hi: 0x8f}, + // Block 0x5a, offset 0x5b + {value: 0x0000, lo: 0x01}, + {value: 0x3c3c, lo: 0x90, hi: 0x90}, + // Block 0x5b, offset 0x5c + {value: 0x000a, lo: 0x03}, + {value: 0x3cc1, lo: 0x80, hi: 0x88}, + {value: 0x3d1b, lo: 0x90, hi: 0x90}, + {value: 0x3d1f, lo: 0x91, hi: 0x91}, + // Block 0x5c, offset 0x5d + {value: 0x0004, lo: 0x0d}, + {value: 0x4463, lo: 0x80, hi: 0x81}, + {value: 0x446c, lo: 0x82, hi: 0x89}, + {value: 0x30a2, lo: 0x8a, hi: 0x8a}, + {value: 0x448d, lo: 0x8b, hi: 0x90}, + {value: 0x44a6, lo: 0x91, hi: 0x92}, + {value: 0x44af, lo: 0x93, hi: 0x93}, + {value: 0x44b4, lo: 0x94, hi: 0x94}, + {value: 0x1b42, lo: 0x95, hi: 0x95}, + {value: 0x44b9, lo: 0x96, hi: 0x96}, + {value: 0x1b52, lo: 0x97, hi: 0x97}, + {value: 0x44bd, lo: 0x98, hi: 0x9b}, + {value: 0x1b66, lo: 0x9c, hi: 0x9c}, + {value: 0x44cd, lo: 0x9d, hi: 0x9d}, } // nfkcDecompLookup: 960 bytes @@ -4367,51 +4608,51 @@ var nfkcDecompLookup = [960]uint8{ // Block 0x1, offset 0x40 // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0x0c2: 0x03, 0x0c3: 0x04, 0x0c4: 0x05, 0x0c5: 0x06, 0x0c6: 0x07, 0x0c7: 0x08, - 0x0c8: 0x09, 0x0ca: 0x0a, 0x0cb: 0x0b, 0x0cd: 0x0c, 0x0ce: 0x0d, 0x0cf: 0x0e, - 0x0d0: 0x0f, 0x0d1: 0x10, 0x0d3: 0x11, 0x0d6: 0x12, - 0x0d8: 0x13, 0x0d9: 0x14, 0x0db: 0x15, + 0x0c2: 0x42, 0x0c3: 0x43, 0x0c4: 0x44, 0x0c5: 0x45, 0x0c6: 0x46, 0x0c7: 0x03, + 0x0c8: 0x47, 0x0ca: 0x48, 0x0cb: 0x49, 0x0cd: 0x4a, 0x0ce: 0x4b, 0x0cf: 0x4c, + 0x0d0: 0x4d, 0x0d1: 0x4e, 0x0d3: 0x4f, 0x0d6: 0x50, + 0x0d8: 0x51, 0x0d9: 0x52, 0x0db: 0x53, 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, 0x0ea: 0x08, 0x0ef: 0x09, 0x0f0: 0x0e, // Block 0x4, offset 0x100 - 0x124: 0x16, 0x125: 0x17, 0x127: 0x18, - 0x128: 0x19, 0x129: 0x1a, 0x12d: 0x1b, 0x12e: 0x1c, 0x12f: 0x1d, - 0x131: 0x1e, 0x133: 0x1f, 0x135: 0x20, 0x137: 0x21, - 0x138: 0x22, 0x13a: 0x23, 0x13b: 0x24, 0x13c: 0x25, 0x13d: 0x26, 0x13e: 0x27, + 0x124: 0x54, 0x125: 0x55, 0x127: 0x56, + 0x128: 0x57, 0x129: 0x58, 0x12d: 0x59, 0x12e: 0x5a, 0x12f: 0x5b, + 0x131: 0x5c, 0x133: 0x5d, 0x135: 0x5e, 0x137: 0x5f, + 0x138: 0x60, 0x13a: 0x61, 0x13b: 0x62, 0x13c: 0x63, 0x13d: 0x64, 0x13e: 0x65, // Block 0x5, offset 0x140 - 0x140: 0x28, 0x143: 0x29, - 0x16c: 0x2a, 0x16d: 0x2b, - 0x174: 0x2c, 0x175: 0x2d, 0x176: 0x2e, - 0x178: 0x2f, 0x179: 0x30, 0x17a: 0x31, 0x17b: 0x32, 0x17c: 0x33, 0x17d: 0x34, 0x17e: 0x35, 0x17f: 0x36, + 0x140: 0x66, 0x143: 0x67, + 0x16c: 0x68, 0x16d: 0x69, + 0x174: 0x6a, 0x175: 0x04, 0x176: 0x6b, + 0x178: 0x6c, 0x179: 0x05, 0x17a: 0x06, 0x17b: 0x07, 0x17c: 0x08, 0x17d: 0x09, 0x17e: 0x0a, 0x17f: 0x0b, // Block 0x6, offset 0x180 - 0x180: 0x37, 0x181: 0x38, 0x182: 0x39, 0x184: 0x3a, 0x185: 0x3b, 0x186: 0x3c, 0x187: 0x3d, - 0x188: 0x3e, 0x189: 0x3f, 0x18a: 0x40, 0x18b: 0x41, 0x18c: 0x42, - 0x191: 0x43, 0x192: 0x44, 0x193: 0x45, - 0x1a8: 0x46, 0x1a9: 0x47, 0x1ab: 0x48, - 0x1b1: 0x49, 0x1b5: 0x4a, - 0x1ba: 0x4b, 0x1bb: 0x4c, 0x1bc: 0x4d, 0x1bd: 0x4e, 0x1be: 0x4f, 0x1bf: 0x50, + 0x180: 0x6d, 0x181: 0x6e, 0x182: 0x0c, 0x184: 0x0d, 0x185: 0x0e, 0x186: 0x6f, 0x187: 0x70, + 0x188: 0x71, 0x189: 0x72, 0x18a: 0x73, 0x18b: 0x74, 0x18c: 0x75, + 0x191: 0x0f, 0x192: 0x10, 0x193: 0x11, + 0x1a8: 0x76, 0x1a9: 0x77, 0x1ab: 0x78, + 0x1b1: 0x79, 0x1b5: 0x7a, + 0x1ba: 0x7b, 0x1bb: 0x7c, 0x1bc: 0x7d, 0x1bd: 0x7e, 0x1be: 0x7f, 0x1bf: 0x80, // Block 0x7, offset 0x1c0 - 0x1c0: 0x51, 0x1c1: 0x52, 0x1c2: 0x53, 0x1c3: 0x54, 0x1c4: 0x55, 0x1c5: 0x56, 0x1c6: 0x57, - 0x1c8: 0x58, 0x1c9: 0x59, 0x1ca: 0x5a, 0x1cb: 0x5b, 0x1cc: 0x5c, 0x1cd: 0x5d, 0x1ce: 0x5e, 0x1cf: 0x5f, + 0x1c0: 0x81, 0x1c1: 0x12, 0x1c2: 0x82, 0x1c3: 0x83, 0x1c4: 0x84, 0x1c5: 0x85, 0x1c6: 0x86, + 0x1c8: 0x13, 0x1c9: 0x14, 0x1ca: 0x15, 0x1cb: 0x87, 0x1cc: 0x16, 0x1cd: 0x17, 0x1ce: 0x18, 0x1cf: 0x19, // Block 0x8, offset 0x200 - 0x21d: 0x60, + 0x21d: 0x88, // Block 0x9, offset 0x240 - 0x264: 0x61, 0x265: 0x62, 0x266: 0x63, 0x267: 0x64, - 0x268: 0x65, 0x269: 0x66, 0x26a: 0x67, 0x26b: 0x68, 0x26c: 0x69, 0x26d: 0x6a, 0x26e: 0x6b, 0x26f: 0x6c, - 0x270: 0x6d, 0x271: 0x6e, 0x272: 0x6f, 0x273: 0x70, 0x274: 0x71, 0x275: 0x72, 0x276: 0x73, 0x277: 0x74, - 0x278: 0x75, 0x279: 0x76, 0x27a: 0x77, 0x27b: 0x78, 0x27c: 0x79, 0x27d: 0x7a, 0x27e: 0x7b, 0x27f: 0x7c, + 0x264: 0x89, 0x265: 0x8a, 0x266: 0x8b, 0x267: 0x8c, + 0x268: 0x8d, 0x269: 0x8e, 0x26a: 0x1a, 0x26b: 0x1b, 0x26c: 0x1c, 0x26d: 0x1d, 0x26e: 0x1e, 0x26f: 0x1f, + 0x270: 0x8f, 0x271: 0x20, 0x272: 0x21, 0x273: 0x22, 0x274: 0x90, 0x275: 0x91, 0x276: 0x92, 0x277: 0x93, + 0x278: 0x94, 0x279: 0x23, 0x27a: 0x24, 0x27b: 0x25, 0x27c: 0x26, 0x27d: 0x27, 0x27e: 0x95, 0x27f: 0x96, // Block 0xa, offset 0x280 - 0x282: 0x7d, + 0x282: 0x97, // Block 0xb, offset 0x2c0 - 0x2c5: 0x7e, 0x2c6: 0x7f, 0x2c7: 0x80, - 0x2d0: 0x81, 0x2d1: 0x82, 0x2d2: 0x83, 0x2d3: 0x84, 0x2d4: 0x85, 0x2d5: 0x86, 0x2d6: 0x87, 0x2d7: 0x88, - 0x2d8: 0x89, 0x2d9: 0x8a, 0x2da: 0x8b, 0x2db: 0x8c, 0x2dc: 0x8d, 0x2dd: 0x8e, 0x2de: 0x8f, 0x2df: 0x90, + 0x2c5: 0x98, 0x2c6: 0x99, 0x2c7: 0x9a, + 0x2d0: 0x28, 0x2d1: 0x29, 0x2d2: 0x2a, 0x2d3: 0x2b, 0x2d4: 0x2c, 0x2d5: 0x2d, 0x2d6: 0x2e, 0x2d7: 0x2f, + 0x2d8: 0x30, 0x2d9: 0x31, 0x2da: 0x32, 0x2db: 0x33, 0x2dc: 0x34, 0x2dd: 0x35, 0x2de: 0x36, 0x2df: 0x37, // Block 0xc, offset 0x300 - 0x304: 0x91, 0x305: 0x92, 0x306: 0x93, - 0x308: 0x94, 0x309: 0x95, + 0x304: 0x38, 0x305: 0x9b, 0x306: 0x9c, + 0x308: 0x39, 0x309: 0x9d, // Block 0xd, offset 0x340 - 0x360: 0x96, 0x361: 0x97, 0x362: 0x98, 0x363: 0x99, 0x364: 0x9a, 0x365: 0x9b, 0x366: 0x9c, 0x367: 0x9d, + 0x360: 0x3a, 0x361: 0x3b, 0x362: 0x3c, 0x363: 0x3d, 0x364: 0x3e, 0x365: 0x3f, 0x366: 0x40, 0x367: 0x41, 0x368: 0x9e, // Block 0xe, offset 0x380 0x391: 0x0a, @@ -4419,7 +4660,7 @@ var nfkcDecompLookup = [960]uint8{ 0x3af: 0x0d, } -var nfkcDecompTrie = trie{nfkcDecompLookup[:], nfkcDecompValues[:]} +var nfkcDecompTrie = trie{nfkcDecompLookup[:], nfkcDecompValues[:], nfkcDecompSparseValues[:], nfkcDecompSparseOffset[:], 66} // recompMap: 7448 bytes (entries only) var recompMap = map[uint32]uint32{ @@ -5356,9 +5597,9 @@ var recompMap = map[uint32]uint32{ 0x10A510BA: 0x110AB, } -// charInfoValues: 10944 entries, 21888 bytes +// charInfoValues: 1024 entries, 2048 bytes // Block 2 is the null block. -var charInfoValues = [10944]uint16{ +var charInfoValues = [1024]uint16{ // Block 0x0, offset 0x0 0x003c: 0x8800, 0x003d: 0x8800, 0x003e: 0x8800, // Block 0x1, offset 0x40 @@ -5374,1121 +5615,1071 @@ var charInfoValues = [10944]uint16{ 0x0076: 0x8800, 0x0077: 0x8800, 0x0078: 0x8800, 0x0079: 0x8800, 0x007a: 0x8800, // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0x00e0: 0x3000, - 0x00e8: 0x3800, - 0x00ea: 0x3000, 0x00ef: 0x3000, - 0x00f2: 0x3000, 0x00f3: 0x3000, 0x00f4: 0x3000, 0x00f5: 0x3000, - 0x00f8: 0x3000, 0x00f9: 0x3000, 0x00fa: 0x3000, - 0x00fc: 0x3000, 0x00fd: 0x3000, 0x00fe: 0x3000, + 0x00c0: 0x1100, 0x00c1: 0x1100, 0x00c2: 0x9900, 0x00c3: 0x1100, 0x00c4: 0x9900, 0x00c5: 0x9900, + 0x00c6: 0x8800, 0x00c7: 0x9900, 0x00c8: 0x1100, 0x00c9: 0x1100, 0x00ca: 0x9900, 0x00cb: 0x1100, + 0x00cc: 0x1100, 0x00cd: 0x1100, 0x00ce: 0x1100, 0x00cf: 0x9900, 0x00d1: 0x1100, + 0x00d2: 0x1100, 0x00d3: 0x1100, 0x00d4: 0x9900, 0x00d5: 0x9900, 0x00d6: 0x9900, + 0x00d8: 0x8800, 0x00d9: 0x1100, 0x00da: 0x1100, 0x00db: 0x1100, 0x00dc: 0x9900, 0x00dd: 0x1100, + 0x00e0: 0x1100, 0x00e1: 0x1100, 0x00e2: 0x9900, 0x00e3: 0x1100, + 0x00e4: 0x9900, 0x00e5: 0x9900, 0x00e6: 0x8800, 0x00e7: 0x9900, 0x00e8: 0x1100, 0x00e9: 0x1100, + 0x00ea: 0x9900, 0x00eb: 0x1100, 0x00ec: 0x1100, 0x00ed: 0x1100, 0x00ee: 0x1100, 0x00ef: 0x9900, + 0x00f1: 0x1100, 0x00f2: 0x1100, 0x00f3: 0x1100, 0x00f4: 0x9900, 0x00f5: 0x9900, + 0x00f6: 0x9900, 0x00f8: 0x8800, 0x00f9: 0x1100, 0x00fa: 0x1100, 0x00fb: 0x1100, + 0x00fc: 0x9900, 0x00fd: 0x1100, 0x00ff: 0x1100, // Block 0x4, offset 0x100 - 0x0100: 0x1100, 0x0101: 0x1100, 0x0102: 0x9900, 0x0103: 0x1100, 0x0104: 0x9900, 0x0105: 0x9900, - 0x0106: 0x8800, 0x0107: 0x9900, 0x0108: 0x1100, 0x0109: 0x1100, 0x010a: 0x9900, 0x010b: 0x1100, - 0x010c: 0x1100, 0x010d: 0x1100, 0x010e: 0x1100, 0x010f: 0x9900, 0x0111: 0x1100, - 0x0112: 0x1100, 0x0113: 0x1100, 0x0114: 0x9900, 0x0115: 0x9900, 0x0116: 0x9900, - 0x0118: 0x8800, 0x0119: 0x1100, 0x011a: 0x1100, 0x011b: 0x1100, 0x011c: 0x9900, 0x011d: 0x1100, - 0x0120: 0x1100, 0x0121: 0x1100, 0x0122: 0x9900, 0x0123: 0x1100, - 0x0124: 0x9900, 0x0125: 0x9900, 0x0126: 0x8800, 0x0127: 0x9900, 0x0128: 0x1100, 0x0129: 0x1100, - 0x012a: 0x9900, 0x012b: 0x1100, 0x012c: 0x1100, 0x012d: 0x1100, 0x012e: 0x1100, 0x012f: 0x9900, - 0x0131: 0x1100, 0x0132: 0x1100, 0x0133: 0x1100, 0x0134: 0x9900, 0x0135: 0x9900, - 0x0136: 0x9900, 0x0138: 0x8800, 0x0139: 0x1100, 0x013a: 0x1100, 0x013b: 0x1100, - 0x013c: 0x9900, 0x013d: 0x1100, 0x013f: 0x1100, + 0x0100: 0x66e6, 0x0101: 0x66e6, 0x0102: 0x66e6, 0x0103: 0x66e6, 0x0104: 0x66e6, 0x0105: 0x00e6, + 0x0106: 0x66e6, 0x0107: 0x66e6, 0x0108: 0x66e6, 0x0109: 0x66e6, 0x010a: 0x66e6, 0x010b: 0x66e6, + 0x010c: 0x66e6, 0x010d: 0x00e6, 0x010e: 0x00e6, 0x010f: 0x66e6, 0x0110: 0x00e6, 0x0111: 0x66e6, + 0x0112: 0x00e6, 0x0113: 0x66e6, 0x0114: 0x66e6, 0x0115: 0x00e8, 0x0116: 0x00dc, 0x0117: 0x00dc, + 0x0118: 0x00dc, 0x0119: 0x00dc, 0x011a: 0x00e8, 0x011b: 0x66d8, 0x011c: 0x00dc, 0x011d: 0x00dc, + 0x011e: 0x00dc, 0x011f: 0x00dc, 0x0120: 0x00dc, 0x0121: 0x00ca, 0x0122: 0x00ca, 0x0123: 0x66dc, + 0x0124: 0x66dc, 0x0125: 0x66dc, 0x0126: 0x66dc, 0x0127: 0x66ca, 0x0128: 0x66ca, 0x0129: 0x00dc, + 0x012a: 0x00dc, 0x012b: 0x00dc, 0x012c: 0x00dc, 0x012d: 0x66dc, 0x012e: 0x66dc, 0x012f: 0x00dc, + 0x0130: 0x66dc, 0x0131: 0x66dc, 0x0132: 0x00dc, 0x0133: 0x00dc, 0x0134: 0x0001, 0x0135: 0x0001, + 0x0136: 0x0001, 0x0137: 0x0001, 0x0138: 0x6601, 0x0139: 0x00dc, 0x013a: 0x00dc, 0x013b: 0x00dc, + 0x013c: 0x00dc, 0x013d: 0x00e6, 0x013e: 0x00e6, 0x013f: 0x00e6, // Block 0x5, offset 0x140 - 0x0140: 0x1100, 0x0141: 0x1100, 0x0142: 0x9900, 0x0143: 0x9900, 0x0144: 0x1100, 0x0145: 0x1100, - 0x0146: 0x1100, 0x0147: 0x1100, 0x0148: 0x1100, 0x0149: 0x1100, 0x014a: 0x1100, 0x014b: 0x1100, - 0x014c: 0x1100, 0x014d: 0x1100, 0x014e: 0x1100, 0x014f: 0x1100, - 0x0152: 0x9900, 0x0153: 0x9900, 0x0154: 0x1100, 0x0155: 0x1100, 0x0156: 0x1100, 0x0157: 0x1100, - 0x0158: 0x1100, 0x0159: 0x1100, 0x015a: 0x1100, 0x015b: 0x1100, 0x015c: 0x1100, 0x015d: 0x1100, - 0x015e: 0x1100, 0x015f: 0x1100, 0x0160: 0x1100, 0x0161: 0x1100, 0x0162: 0x1100, 0x0163: 0x1100, - 0x0164: 0x1100, 0x0165: 0x1100, 0x0168: 0x1100, 0x0169: 0x1100, - 0x016a: 0x1100, 0x016b: 0x1100, 0x016c: 0x1100, 0x016d: 0x1100, 0x016e: 0x1100, 0x016f: 0x1100, - 0x0170: 0x1100, 0x0172: 0x3000, 0x0173: 0x3000, 0x0174: 0x1100, 0x0175: 0x1100, - 0x0176: 0x1100, 0x0177: 0x1100, 0x0179: 0x1100, 0x017a: 0x1100, 0x017b: 0x1100, - 0x017c: 0x1100, 0x017d: 0x1100, 0x017e: 0x1100, 0x017f: 0x3000, + 0x0140: 0x33e6, 0x0141: 0x33e6, 0x0142: 0x66e6, 0x0143: 0x33e6, 0x0144: 0x33e6, 0x0145: 0x66f0, + 0x0146: 0x00e6, 0x0147: 0x00dc, 0x0148: 0x00dc, 0x0149: 0x00dc, 0x014a: 0x00e6, 0x014b: 0x00e6, + 0x014c: 0x00e6, 0x014d: 0x00dc, 0x014e: 0x00dc, 0x0150: 0x00e6, 0x0151: 0x00e6, + 0x0152: 0x00e6, 0x0153: 0x00dc, 0x0154: 0x00dc, 0x0155: 0x00dc, 0x0156: 0x00dc, 0x0157: 0x00e6, + 0x0158: 0x00e8, 0x0159: 0x00dc, 0x015a: 0x00dc, 0x015b: 0x00e6, 0x015c: 0x00e9, 0x015d: 0x00ea, + 0x015e: 0x00ea, 0x015f: 0x00e9, 0x0160: 0x00ea, 0x0161: 0x00ea, 0x0162: 0x00e9, 0x0163: 0x00e6, + 0x0164: 0x00e6, 0x0165: 0x00e6, 0x0166: 0x00e6, 0x0167: 0x00e6, 0x0168: 0x00e6, 0x0169: 0x00e6, + 0x016a: 0x00e6, 0x016b: 0x00e6, 0x016c: 0x00e6, 0x016d: 0x00e6, 0x016e: 0x00e6, 0x016f: 0x00e6, + 0x0174: 0x3300, + 0x017a: 0x3000, + 0x017e: 0x3300, // Block 0x6, offset 0x180 - 0x0180: 0x3000, 0x0183: 0x1100, 0x0184: 0x1100, 0x0185: 0x1100, - 0x0186: 0x1100, 0x0187: 0x1100, 0x0188: 0x1100, 0x0189: 0x3000, - 0x018c: 0x9900, 0x018d: 0x9900, 0x018e: 0x1100, 0x018f: 0x1100, 0x0190: 0x1100, 0x0191: 0x1100, - 0x0194: 0x1100, 0x0195: 0x1100, 0x0196: 0x1100, 0x0197: 0x1100, - 0x0198: 0x1100, 0x0199: 0x1100, 0x019a: 0x9900, 0x019b: 0x9900, 0x019c: 0x1100, 0x019d: 0x1100, - 0x019e: 0x1100, 0x019f: 0x1100, 0x01a0: 0x9900, 0x01a1: 0x9900, 0x01a2: 0x1100, 0x01a3: 0x1100, - 0x01a4: 0x1100, 0x01a5: 0x1100, 0x01a8: 0x9900, 0x01a9: 0x9900, - 0x01aa: 0x9900, 0x01ab: 0x9900, 0x01ac: 0x1100, 0x01ad: 0x1100, 0x01ae: 0x1100, 0x01af: 0x1100, - 0x01b0: 0x1100, 0x01b1: 0x1100, 0x01b2: 0x1100, 0x01b3: 0x1100, 0x01b4: 0x1100, 0x01b5: 0x1100, - 0x01b6: 0x1100, 0x01b7: 0x1100, 0x01b8: 0x1100, 0x01b9: 0x1100, 0x01ba: 0x1100, 0x01bb: 0x1100, - 0x01bc: 0x1100, 0x01bd: 0x1100, 0x01be: 0x1100, 0x01bf: 0x3800, + 0x0184: 0x3000, 0x0185: 0x3100, + 0x0186: 0x1100, 0x0187: 0x3300, 0x0188: 0x1100, 0x0189: 0x1100, 0x018a: 0x1100, + 0x018c: 0x1100, 0x018e: 0x1100, 0x018f: 0x1100, 0x0190: 0x1100, 0x0191: 0x8800, + 0x0195: 0x8800, 0x0197: 0x8800, + 0x0199: 0x8800, + 0x019f: 0x8800, 0x01a1: 0x8800, + 0x01a5: 0x8800, 0x01a9: 0x8800, + 0x01aa: 0x1100, 0x01ab: 0x1100, 0x01ac: 0x9900, 0x01ad: 0x1100, 0x01ae: 0x9900, 0x01af: 0x1100, + 0x01b0: 0x1100, 0x01b1: 0x8800, 0x01b5: 0x8800, + 0x01b7: 0x8800, 0x01b9: 0x8800, + 0x01bf: 0x8800, // Block 0x7, offset 0x1c0 - 0x01e0: 0x9900, 0x01e1: 0x9900, - 0x01ef: 0x9900, - 0x01f0: 0x9900, - 0x01f7: 0x8800, + 0x01c0: 0x1100, 0x01c1: 0x1100, 0x01c3: 0x1100, + 0x01c6: 0x8800, 0x01c7: 0x1100, + 0x01cc: 0x1100, 0x01cd: 0x1100, 0x01ce: 0x1100, 0x01d0: 0x8800, + 0x01d3: 0x8800, 0x01d5: 0x8800, 0x01d6: 0x8800, 0x01d7: 0x8800, + 0x01d8: 0x8800, 0x01d9: 0x1100, 0x01da: 0x8800, + 0x01de: 0x8800, 0x01e3: 0x8800, + 0x01e7: 0x8800, + 0x01eb: 0x8800, 0x01ed: 0x8800, + 0x01f0: 0x8800, 0x01f3: 0x8800, 0x01f5: 0x8800, + 0x01f6: 0x8800, 0x01f7: 0x8800, 0x01f8: 0x8800, 0x01f9: 0x1100, 0x01fa: 0x8800, + 0x01fe: 0x8800, // Block 0x8, offset 0x200 - 0x0204: 0x3000, 0x0205: 0x3000, - 0x0206: 0x3000, 0x0207: 0x3000, 0x0208: 0x3000, 0x0209: 0x3000, 0x020a: 0x3000, 0x020b: 0x3000, - 0x020c: 0x3000, 0x020d: 0x1100, 0x020e: 0x1100, 0x020f: 0x1100, 0x0210: 0x1100, 0x0211: 0x1100, - 0x0212: 0x1100, 0x0213: 0x1100, 0x0214: 0x1100, 0x0215: 0x1100, 0x0216: 0x1100, 0x0217: 0x1100, - 0x0218: 0x1100, 0x0219: 0x1100, 0x021a: 0x1100, 0x021b: 0x1100, 0x021c: 0x1100, - 0x021e: 0x1100, 0x021f: 0x1100, 0x0220: 0x1100, 0x0221: 0x1100, 0x0222: 0x1100, 0x0223: 0x1100, - 0x0226: 0x1100, 0x0227: 0x1100, 0x0228: 0x1100, 0x0229: 0x1100, - 0x022a: 0x9900, 0x022b: 0x9900, 0x022c: 0x1100, 0x022d: 0x1100, 0x022e: 0x1100, 0x022f: 0x1100, - 0x0230: 0x1100, 0x0231: 0x3000, 0x0232: 0x3000, 0x0233: 0x3000, 0x0234: 0x1100, 0x0235: 0x1100, - 0x0238: 0x1100, 0x0239: 0x1100, 0x023a: 0x1100, 0x023b: 0x1100, - 0x023c: 0x1100, 0x023d: 0x1100, 0x023e: 0x1100, 0x023f: 0x1100, + 0x0207: 0x3000, + 0x0211: 0x00dc, + 0x0212: 0x00e6, 0x0213: 0x00e6, 0x0214: 0x00e6, 0x0215: 0x00e6, 0x0216: 0x00dc, 0x0217: 0x00e6, + 0x0218: 0x00e6, 0x0219: 0x00e6, 0x021a: 0x00de, 0x021b: 0x00dc, 0x021c: 0x00e6, 0x021d: 0x00e6, + 0x021e: 0x00e6, 0x021f: 0x00e6, 0x0220: 0x00e6, 0x0221: 0x00e6, 0x0222: 0x00dc, 0x0223: 0x00dc, + 0x0224: 0x00dc, 0x0225: 0x00dc, 0x0226: 0x00dc, 0x0227: 0x00dc, 0x0228: 0x00e6, 0x0229: 0x00e6, + 0x022a: 0x00dc, 0x022b: 0x00e6, 0x022c: 0x00e6, 0x022d: 0x00de, 0x022e: 0x00e4, 0x022f: 0x00e6, + 0x0230: 0x000a, 0x0231: 0x000b, 0x0232: 0x000c, 0x0233: 0x000d, 0x0234: 0x000e, 0x0235: 0x000f, + 0x0236: 0x0010, 0x0237: 0x0011, 0x0238: 0x0012, 0x0239: 0x0013, 0x023a: 0x0013, 0x023b: 0x0014, + 0x023c: 0x0015, 0x023d: 0x0016, 0x023f: 0x0017, // Block 0x9, offset 0x240 - 0x0240: 0x1100, 0x0241: 0x1100, 0x0242: 0x1100, 0x0243: 0x1100, 0x0244: 0x1100, 0x0245: 0x1100, - 0x0246: 0x1100, 0x0247: 0x1100, 0x0248: 0x1100, 0x0249: 0x1100, 0x024a: 0x1100, 0x024b: 0x1100, - 0x024c: 0x1100, 0x024d: 0x1100, 0x024e: 0x1100, 0x024f: 0x1100, 0x0250: 0x1100, 0x0251: 0x1100, - 0x0252: 0x1100, 0x0253: 0x1100, 0x0254: 0x1100, 0x0255: 0x1100, 0x0256: 0x1100, 0x0257: 0x1100, - 0x0258: 0x1100, 0x0259: 0x1100, 0x025a: 0x1100, 0x025b: 0x1100, - 0x025e: 0x1100, 0x025f: 0x1100, - 0x0266: 0x9900, 0x0267: 0x9900, 0x0268: 0x9900, 0x0269: 0x9900, - 0x026a: 0x1100, 0x026b: 0x1100, 0x026c: 0x1100, 0x026d: 0x1100, 0x026e: 0x9900, 0x026f: 0x9900, - 0x0270: 0x1100, 0x0271: 0x1100, 0x0272: 0x1100, 0x0273: 0x1100, + 0x0248: 0x8800, 0x024a: 0x8800, 0x024b: 0x001b, + 0x024c: 0x001c, 0x024d: 0x001d, 0x024e: 0x001e, 0x024f: 0x001f, 0x0250: 0x0020, 0x0251: 0x0021, + 0x0252: 0x0022, 0x0253: 0x66e6, 0x0254: 0x66e6, 0x0255: 0x66dc, 0x0256: 0x00dc, 0x0257: 0x00e6, + 0x0258: 0x00e6, 0x0259: 0x00e6, 0x025a: 0x00e6, 0x025b: 0x00e6, 0x025c: 0x00dc, 0x025d: 0x00e6, + 0x025e: 0x00e6, 0x025f: 0x00dc, + 0x0270: 0x0023, 0x0275: 0x3000, + 0x0276: 0x3000, 0x0277: 0x3000, 0x0278: 0x3000, // Block 0xa, offset 0x280 - 0x0292: 0x8800, - 0x02b0: 0x3000, 0x02b1: 0x3000, 0x02b2: 0x3000, 0x02b3: 0x3000, 0x02b4: 0x3000, 0x02b5: 0x3000, - 0x02b6: 0x3000, 0x02b7: 0x3000, 0x02b8: 0x3000, + 0x0280: 0x9900, 0x0281: 0x9900, 0x0282: 0x1100, 0x0283: 0x1100, 0x0284: 0x1100, 0x0285: 0x1100, + 0x0288: 0x9900, 0x0289: 0x9900, 0x028a: 0x1100, 0x028b: 0x1100, + 0x028c: 0x1100, 0x028d: 0x1100, 0x0290: 0x9900, 0x0291: 0x9900, + 0x0292: 0x1100, 0x0293: 0x1100, 0x0294: 0x1100, 0x0295: 0x1100, 0x0296: 0x1100, 0x0297: 0x1100, + 0x0299: 0x9900, 0x029b: 0x1100, 0x029d: 0x1100, + 0x029f: 0x1100, 0x02a0: 0x9900, 0x02a1: 0x9900, 0x02a2: 0x9900, 0x02a3: 0x9900, + 0x02a4: 0x9900, 0x02a5: 0x9900, 0x02a6: 0x9900, 0x02a7: 0x9900, 0x02a8: 0x9900, 0x02a9: 0x9900, + 0x02aa: 0x9900, 0x02ab: 0x9900, 0x02ac: 0x9900, 0x02ad: 0x9900, 0x02ae: 0x9900, 0x02af: 0x9900, + 0x02b0: 0x9900, 0x02b1: 0x3300, 0x02b2: 0x1100, 0x02b3: 0x3300, 0x02b4: 0x9900, 0x02b5: 0x3300, + 0x02b6: 0x1100, 0x02b7: 0x3300, 0x02b8: 0x1100, 0x02b9: 0x3300, 0x02ba: 0x1100, 0x02bb: 0x3300, + 0x02bc: 0x9900, 0x02bd: 0x3300, // Block 0xb, offset 0x2c0 - 0x02d8: 0x3000, 0x02d9: 0x3000, 0x02da: 0x3000, 0x02db: 0x3000, 0x02dc: 0x3000, 0x02dd: 0x3000, - 0x02e0: 0x3000, 0x02e1: 0x3000, 0x02e2: 0x3000, 0x02e3: 0x3000, - 0x02e4: 0x3000, + 0x02c0: 0x3000, 0x02c1: 0x3100, 0x02c2: 0x1100, 0x02c3: 0x1100, 0x02c4: 0x1100, + 0x02c6: 0x9900, 0x02c7: 0x1100, 0x02c8: 0x1100, 0x02c9: 0x3300, 0x02ca: 0x1100, 0x02cb: 0x3300, + 0x02cc: 0x1100, 0x02cd: 0x3100, 0x02ce: 0x3100, 0x02cf: 0x3100, 0x02d0: 0x1100, 0x02d1: 0x1100, + 0x02d2: 0x1100, 0x02d3: 0x3300, 0x02d6: 0x1100, 0x02d7: 0x1100, + 0x02d8: 0x1100, 0x02d9: 0x1100, 0x02da: 0x1100, 0x02db: 0x3300, 0x02dd: 0x3100, + 0x02de: 0x3100, 0x02df: 0x3100, 0x02e0: 0x1100, 0x02e1: 0x1100, 0x02e2: 0x1100, 0x02e3: 0x3300, + 0x02e4: 0x1100, 0x02e5: 0x1100, 0x02e6: 0x1100, 0x02e7: 0x1100, 0x02e8: 0x1100, 0x02e9: 0x1100, + 0x02ea: 0x1100, 0x02eb: 0x3300, 0x02ec: 0x1100, 0x02ed: 0x3100, 0x02ee: 0x3300, 0x02ef: 0x3300, + 0x02f2: 0x1100, 0x02f3: 0x1100, 0x02f4: 0x1100, + 0x02f6: 0x9900, 0x02f7: 0x1100, 0x02f8: 0x1100, 0x02f9: 0x3300, 0x02fa: 0x1100, 0x02fb: 0x3300, + 0x02fc: 0x1100, 0x02fd: 0x3300, 0x02fe: 0x3800, // Block 0xc, offset 0x300 - 0x0300: 0x66e6, 0x0301: 0x66e6, 0x0302: 0x66e6, 0x0303: 0x66e6, 0x0304: 0x66e6, 0x0305: 0x00e6, - 0x0306: 0x66e6, 0x0307: 0x66e6, 0x0308: 0x66e6, 0x0309: 0x66e6, 0x030a: 0x66e6, 0x030b: 0x66e6, - 0x030c: 0x66e6, 0x030d: 0x00e6, 0x030e: 0x00e6, 0x030f: 0x66e6, 0x0310: 0x00e6, 0x0311: 0x66e6, - 0x0312: 0x00e6, 0x0313: 0x66e6, 0x0314: 0x66e6, 0x0315: 0x00e8, 0x0316: 0x00dc, 0x0317: 0x00dc, - 0x0318: 0x00dc, 0x0319: 0x00dc, 0x031a: 0x00e8, 0x031b: 0x66d8, 0x031c: 0x00dc, 0x031d: 0x00dc, - 0x031e: 0x00dc, 0x031f: 0x00dc, 0x0320: 0x00dc, 0x0321: 0x00ca, 0x0322: 0x00ca, 0x0323: 0x66dc, - 0x0324: 0x66dc, 0x0325: 0x66dc, 0x0326: 0x66dc, 0x0327: 0x66ca, 0x0328: 0x66ca, 0x0329: 0x00dc, - 0x032a: 0x00dc, 0x032b: 0x00dc, 0x032c: 0x00dc, 0x032d: 0x66dc, 0x032e: 0x66dc, 0x032f: 0x00dc, - 0x0330: 0x66dc, 0x0331: 0x66dc, 0x0332: 0x00dc, 0x0333: 0x00dc, 0x0334: 0x0001, 0x0335: 0x0001, - 0x0336: 0x0001, 0x0337: 0x0001, 0x0338: 0x6601, 0x0339: 0x00dc, 0x033a: 0x00dc, 0x033b: 0x00dc, - 0x033c: 0x00dc, 0x033d: 0x00e6, 0x033e: 0x00e6, 0x033f: 0x00e6, + 0x0301: 0x1100, 0x0303: 0x8800, 0x0304: 0x1100, 0x0305: 0x8800, + 0x0307: 0x1100, 0x0308: 0x8800, 0x0309: 0x1100, + 0x030d: 0x8800, + 0x0320: 0x1100, 0x0321: 0x8800, 0x0322: 0x1100, + 0x0324: 0x8800, 0x0325: 0x8800, + 0x032d: 0x1100, 0x032e: 0x1100, 0x032f: 0x1100, + 0x0330: 0x1100, 0x0331: 0x1100, 0x0332: 0x8800, 0x0333: 0x8800, 0x0334: 0x1100, 0x0335: 0x1100, + 0x0336: 0x8800, 0x0337: 0x8800, 0x0338: 0x1100, 0x0339: 0x1100, 0x033a: 0x8800, 0x033b: 0x8800, + 0x033c: 0x8800, 0x033d: 0x8800, // Block 0xd, offset 0x340 - 0x0340: 0x33e6, 0x0341: 0x33e6, 0x0342: 0x66e6, 0x0343: 0x33e6, 0x0344: 0x33e6, 0x0345: 0x66f0, - 0x0346: 0x00e6, 0x0347: 0x00dc, 0x0348: 0x00dc, 0x0349: 0x00dc, 0x034a: 0x00e6, 0x034b: 0x00e6, - 0x034c: 0x00e6, 0x034d: 0x00dc, 0x034e: 0x00dc, 0x0350: 0x00e6, 0x0351: 0x00e6, - 0x0352: 0x00e6, 0x0353: 0x00dc, 0x0354: 0x00dc, 0x0355: 0x00dc, 0x0356: 0x00dc, 0x0357: 0x00e6, - 0x0358: 0x00e8, 0x0359: 0x00dc, 0x035a: 0x00dc, 0x035b: 0x00e6, 0x035c: 0x00e9, 0x035d: 0x00ea, - 0x035e: 0x00ea, 0x035f: 0x00e9, 0x0360: 0x00ea, 0x0361: 0x00ea, 0x0362: 0x00e9, 0x0363: 0x00e6, - 0x0364: 0x00e6, 0x0365: 0x00e6, 0x0366: 0x00e6, 0x0367: 0x00e6, 0x0368: 0x00e6, 0x0369: 0x00e6, - 0x036a: 0x00e6, 0x036b: 0x00e6, 0x036c: 0x00e6, 0x036d: 0x00e6, 0x036e: 0x00e6, 0x036f: 0x00e6, - 0x0374: 0x3300, - 0x037a: 0x3000, - 0x037e: 0x3300, + 0x0346: 0x8800, 0x034b: 0x8800, + 0x034c: 0x1100, 0x034d: 0x8800, 0x034e: 0x1100, 0x034f: 0x8800, 0x0350: 0x1100, 0x0351: 0x8800, + 0x0352: 0x1100, 0x0353: 0x8800, 0x0354: 0x1100, 0x0355: 0x8800, 0x0356: 0x1100, 0x0357: 0x8800, + 0x0358: 0x1100, 0x0359: 0x8800, 0x035a: 0x1100, 0x035b: 0x8800, 0x035c: 0x1100, 0x035d: 0x8800, + 0x035e: 0x1100, 0x035f: 0x8800, 0x0360: 0x1100, 0x0361: 0x8800, 0x0362: 0x1100, + 0x0364: 0x8800, 0x0365: 0x1100, 0x0366: 0x8800, 0x0367: 0x1100, 0x0368: 0x8800, 0x0369: 0x1100, + 0x036f: 0x8800, + 0x0370: 0x1100, 0x0371: 0x1100, 0x0372: 0x8800, 0x0373: 0x1100, 0x0374: 0x1100, 0x0375: 0x8800, + 0x0376: 0x1100, 0x0377: 0x1100, 0x0378: 0x8800, 0x0379: 0x1100, 0x037a: 0x1100, 0x037b: 0x8800, + 0x037c: 0x1100, 0x037d: 0x1100, // Block 0xe, offset 0x380 - 0x0384: 0x3000, 0x0385: 0x3100, - 0x0386: 0x1100, 0x0387: 0x3300, 0x0388: 0x1100, 0x0389: 0x1100, 0x038a: 0x1100, - 0x038c: 0x1100, 0x038e: 0x1100, 0x038f: 0x1100, 0x0390: 0x1100, 0x0391: 0x8800, - 0x0395: 0x8800, 0x0397: 0x8800, - 0x0399: 0x8800, - 0x039f: 0x8800, 0x03a1: 0x8800, - 0x03a5: 0x8800, 0x03a9: 0x8800, - 0x03aa: 0x1100, 0x03ab: 0x1100, 0x03ac: 0x9900, 0x03ad: 0x1100, 0x03ae: 0x9900, 0x03af: 0x1100, - 0x03b0: 0x1100, 0x03b1: 0x8800, 0x03b5: 0x8800, - 0x03b7: 0x8800, 0x03b9: 0x8800, - 0x03bf: 0x8800, + 0x0394: 0x1100, + 0x0399: 0x6608, 0x039a: 0x6608, 0x039b: 0x3000, 0x039c: 0x3000, 0x039d: 0x8800, + 0x039e: 0x1100, 0x039f: 0x3000, + 0x03a6: 0x8800, + 0x03ab: 0x8800, 0x03ac: 0x1100, 0x03ad: 0x8800, 0x03ae: 0x1100, 0x03af: 0x8800, + 0x03b0: 0x1100, 0x03b1: 0x8800, 0x03b2: 0x1100, 0x03b3: 0x8800, 0x03b4: 0x1100, 0x03b5: 0x8800, + 0x03b6: 0x1100, 0x03b7: 0x8800, 0x03b8: 0x1100, 0x03b9: 0x8800, 0x03ba: 0x1100, 0x03bb: 0x8800, + 0x03bc: 0x1100, 0x03bd: 0x8800, 0x03be: 0x1100, 0x03bf: 0x8800, // Block 0xf, offset 0x3c0 - 0x03c1: 0x8800, 0x03c5: 0x8800, - 0x03c9: 0x8800, 0x03ca: 0x9900, 0x03cb: 0x9900, - 0x03cc: 0x1100, 0x03cd: 0x1100, 0x03ce: 0x9900, 0x03d0: 0x3000, 0x03d1: 0x3000, - 0x03d2: 0x3800, 0x03d3: 0x3100, 0x03d4: 0x3100, 0x03d5: 0x3000, 0x03d6: 0x3000, - 0x03f0: 0x3000, 0x03f1: 0x3000, 0x03f2: 0x3000, 0x03f4: 0x3000, 0x03f5: 0x3000, - 0x03f9: 0x3000, - // Block 0x10, offset 0x400 - 0x0400: 0x1100, 0x0401: 0x1100, 0x0403: 0x1100, - 0x0406: 0x8800, 0x0407: 0x1100, - 0x040c: 0x1100, 0x040d: 0x1100, 0x040e: 0x1100, 0x0410: 0x8800, - 0x0413: 0x8800, 0x0415: 0x8800, 0x0416: 0x8800, 0x0417: 0x8800, - 0x0418: 0x8800, 0x0419: 0x1100, 0x041a: 0x8800, - 0x041e: 0x8800, 0x0423: 0x8800, - 0x0427: 0x8800, - 0x042b: 0x8800, 0x042d: 0x8800, - 0x0430: 0x8800, 0x0433: 0x8800, 0x0435: 0x8800, - 0x0436: 0x8800, 0x0437: 0x8800, 0x0438: 0x8800, 0x0439: 0x1100, 0x043a: 0x8800, - 0x043e: 0x8800, - // Block 0x11, offset 0x440 - 0x0443: 0x8800, - 0x0447: 0x8800, 0x044b: 0x8800, - 0x044d: 0x8800, 0x0450: 0x1100, 0x0451: 0x1100, - 0x0453: 0x1100, 0x0456: 0x8800, 0x0457: 0x1100, - 0x045c: 0x1100, 0x045d: 0x1100, - 0x045e: 0x1100, - 0x0474: 0x8800, 0x0475: 0x8800, - 0x0476: 0x1100, 0x0477: 0x1100, - // Block 0x12, offset 0x480 - 0x0483: 0x00e6, 0x0484: 0x00e6, 0x0485: 0x00e6, - 0x0486: 0x00e6, 0x0487: 0x00e6, - // Block 0x13, offset 0x4c0 - 0x04c1: 0x1100, 0x04c2: 0x1100, - 0x04d0: 0x1100, 0x04d1: 0x1100, - 0x04d2: 0x1100, 0x04d3: 0x1100, 0x04d6: 0x1100, 0x04d7: 0x1100, - 0x04d8: 0x8800, 0x04d9: 0x8800, 0x04da: 0x1100, 0x04db: 0x1100, 0x04dc: 0x1100, 0x04dd: 0x1100, - 0x04de: 0x1100, 0x04df: 0x1100, 0x04e2: 0x1100, 0x04e3: 0x1100, - 0x04e4: 0x1100, 0x04e5: 0x1100, 0x04e6: 0x1100, 0x04e7: 0x1100, 0x04e8: 0x8800, 0x04e9: 0x8800, - 0x04ea: 0x1100, 0x04eb: 0x1100, 0x04ec: 0x1100, 0x04ed: 0x1100, 0x04ee: 0x1100, 0x04ef: 0x1100, - 0x04f0: 0x1100, 0x04f1: 0x1100, 0x04f2: 0x1100, 0x04f3: 0x1100, 0x04f4: 0x1100, 0x04f5: 0x1100, - 0x04f8: 0x1100, 0x04f9: 0x1100, - // Block 0x14, offset 0x500 - 0x0507: 0x3000, - 0x0511: 0x00dc, - 0x0512: 0x00e6, 0x0513: 0x00e6, 0x0514: 0x00e6, 0x0515: 0x00e6, 0x0516: 0x00dc, 0x0517: 0x00e6, - 0x0518: 0x00e6, 0x0519: 0x00e6, 0x051a: 0x00de, 0x051b: 0x00dc, 0x051c: 0x00e6, 0x051d: 0x00e6, - 0x051e: 0x00e6, 0x051f: 0x00e6, 0x0520: 0x00e6, 0x0521: 0x00e6, 0x0522: 0x00dc, 0x0523: 0x00dc, - 0x0524: 0x00dc, 0x0525: 0x00dc, 0x0526: 0x00dc, 0x0527: 0x00dc, 0x0528: 0x00e6, 0x0529: 0x00e6, - 0x052a: 0x00dc, 0x052b: 0x00e6, 0x052c: 0x00e6, 0x052d: 0x00de, 0x052e: 0x00e4, 0x052f: 0x00e6, - 0x0530: 0x000a, 0x0531: 0x000b, 0x0532: 0x000c, 0x0533: 0x000d, 0x0534: 0x000e, 0x0535: 0x000f, - 0x0536: 0x0010, 0x0537: 0x0011, 0x0538: 0x0012, 0x0539: 0x0013, 0x053a: 0x0013, 0x053b: 0x0014, - 0x053c: 0x0015, 0x053d: 0x0016, 0x053f: 0x0017, - // Block 0x15, offset 0x540 - 0x0541: 0x0018, 0x0542: 0x0019, 0x0544: 0x00e6, 0x0545: 0x00dc, - 0x0547: 0x0012, - // Block 0x16, offset 0x580 - 0x0590: 0x00e6, 0x0591: 0x00e6, - 0x0592: 0x00e6, 0x0593: 0x00e6, 0x0594: 0x00e6, 0x0595: 0x00e6, 0x0596: 0x00e6, 0x0597: 0x00e6, - 0x0598: 0x001e, 0x0599: 0x001f, 0x059a: 0x0020, - 0x05a2: 0x1100, 0x05a3: 0x1100, - 0x05a4: 0x1100, 0x05a5: 0x1100, 0x05a6: 0x1100, 0x05a7: 0x8800, - // Block 0x17, offset 0x5c0 - 0x05c8: 0x8800, 0x05ca: 0x8800, 0x05cb: 0x001b, - 0x05cc: 0x001c, 0x05cd: 0x001d, 0x05ce: 0x001e, 0x05cf: 0x001f, 0x05d0: 0x0020, 0x05d1: 0x0021, - 0x05d2: 0x0022, 0x05d3: 0x66e6, 0x05d4: 0x66e6, 0x05d5: 0x66dc, 0x05d6: 0x00dc, 0x05d7: 0x00e6, - 0x05d8: 0x00e6, 0x05d9: 0x00e6, 0x05da: 0x00e6, 0x05db: 0x00e6, 0x05dc: 0x00dc, 0x05dd: 0x00e6, - 0x05de: 0x00e6, 0x05df: 0x00dc, - 0x05f0: 0x0023, 0x05f5: 0x3000, - 0x05f6: 0x3000, 0x05f7: 0x3000, 0x05f8: 0x3000, - // Block 0x18, offset 0x600 - 0x0600: 0x1100, 0x0601: 0x8800, 0x0602: 0x1100, - 0x0612: 0x8800, 0x0613: 0x1100, 0x0615: 0x8800, 0x0616: 0x00e6, 0x0617: 0x00e6, - 0x0618: 0x00e6, 0x0619: 0x00e6, 0x061a: 0x00e6, 0x061b: 0x00e6, 0x061c: 0x00e6, - 0x061f: 0x00e6, 0x0620: 0x00e6, 0x0621: 0x00e6, 0x0622: 0x00e6, 0x0623: 0x00dc, - 0x0624: 0x00e6, 0x0627: 0x00e6, 0x0628: 0x00e6, - 0x062a: 0x00dc, 0x062b: 0x00e6, 0x062c: 0x00e6, 0x062d: 0x00dc, - // Block 0x19, offset 0x640 - 0x0651: 0x0024, - 0x0670: 0x00e6, 0x0671: 0x00dc, 0x0672: 0x00e6, 0x0673: 0x00e6, 0x0674: 0x00dc, 0x0675: 0x00e6, - 0x0676: 0x00e6, 0x0677: 0x00dc, 0x0678: 0x00dc, 0x0679: 0x00dc, 0x067a: 0x00e6, 0x067b: 0x00dc, - 0x067c: 0x00dc, 0x067d: 0x00e6, 0x067e: 0x00dc, 0x067f: 0x00e6, - // Block 0x1a, offset 0x680 - 0x0680: 0x00e6, 0x0681: 0x00e6, 0x0682: 0x00dc, 0x0683: 0x00e6, 0x0684: 0x00dc, 0x0685: 0x00e6, - 0x0686: 0x00dc, 0x0687: 0x00e6, 0x0688: 0x00dc, 0x0689: 0x00e6, 0x068a: 0x00e6, - // Block 0x1b, offset 0x6c0 - 0x06eb: 0x00e6, 0x06ec: 0x00e6, 0x06ed: 0x00e6, 0x06ee: 0x00e6, 0x06ef: 0x00e6, - 0x06f0: 0x00e6, 0x06f1: 0x00e6, 0x06f2: 0x00dc, 0x06f3: 0x00e6, - // Block 0x1c, offset 0x700 - 0x0716: 0x00e6, 0x0717: 0x00e6, - 0x0718: 0x00e6, 0x0719: 0x00e6, 0x071b: 0x00e6, 0x071c: 0x00e6, 0x071d: 0x00e6, - 0x071e: 0x00e6, 0x071f: 0x00e6, 0x0720: 0x00e6, 0x0721: 0x00e6, 0x0722: 0x00e6, 0x0723: 0x00e6, - 0x0725: 0x00e6, 0x0726: 0x00e6, 0x0727: 0x00e6, 0x0729: 0x00e6, - 0x072a: 0x00e6, 0x072b: 0x00e6, 0x072c: 0x00e6, 0x072d: 0x00e6, - // Block 0x1d, offset 0x740 - 0x0759: 0x00dc, 0x075a: 0x00dc, 0x075b: 0x00dc, - // Block 0x1e, offset 0x780 - 0x07a8: 0x8800, 0x07a9: 0x1100, - 0x07b0: 0x8800, 0x07b1: 0x1100, 0x07b3: 0x8800, 0x07b4: 0x1100, - 0x07bc: 0x6607, - // Block 0x1f, offset 0x7c0 - 0x07cd: 0x0009, 0x07d1: 0x00e6, - 0x07d2: 0x00dc, 0x07d3: 0x00e6, 0x07d4: 0x00e6, - 0x07d8: 0x3300, 0x07d9: 0x3300, 0x07da: 0x3300, 0x07db: 0x3300, 0x07dc: 0x3300, 0x07dd: 0x3300, - 0x07de: 0x3300, 0x07df: 0x3300, - // Block 0x20, offset 0x800 - 0x083c: 0x0007, 0x083e: 0x6600, - // Block 0x21, offset 0x840 - 0x0847: 0x8800, 0x084b: 0x1100, - 0x084c: 0x1100, 0x084d: 0x0009, - 0x0857: 0x6600, - 0x085c: 0x3300, 0x085d: 0x3300, - 0x085f: 0x3300, - // Block 0x22, offset 0x880 - 0x08b3: 0x3300, - 0x08b6: 0x3300, - 0x08bc: 0x0007, - // Block 0x23, offset 0x8c0 - 0x08cd: 0x0009, - 0x08d9: 0x3300, 0x08da: 0x3300, 0x08db: 0x3300, - 0x08de: 0x3300, - // Block 0x24, offset 0x900 - 0x093c: 0x0007, - // Block 0x25, offset 0x940 - 0x094d: 0x0009, - // Block 0x26, offset 0x980 - 0x0987: 0x8800, 0x0988: 0x1100, 0x098b: 0x1100, - 0x098c: 0x1100, 0x098d: 0x0009, - 0x0996: 0x6600, 0x0997: 0x6600, - 0x099c: 0x3300, 0x099d: 0x3300, - // Block 0x27, offset 0x9c0 - 0x09d2: 0x8800, 0x09d4: 0x1100, - 0x09fe: 0x6600, - // Block 0x28, offset 0xa00 - 0x0a06: 0x8800, 0x0a07: 0x8800, 0x0a0a: 0x1100, 0x0a0b: 0x1100, - 0x0a0c: 0x1100, 0x0a0d: 0x0009, - 0x0a17: 0x6600, - // Block 0x29, offset 0xa40 - 0x0a46: 0x8800, 0x0a48: 0x1100, - 0x0a4d: 0x0009, - 0x0a55: 0x0054, 0x0a56: 0x665b, - // Block 0x2a, offset 0xa80 - 0x0abc: 0x0007, 0x0abf: 0x8800, - // Block 0x2b, offset 0xac0 - 0x0ac0: 0x1100, 0x0ac2: 0x6600, - 0x0ac6: 0x8800, 0x0ac7: 0x1100, 0x0ac8: 0x1100, 0x0aca: 0x9900, 0x0acb: 0x1100, - 0x0acd: 0x0009, - 0x0ad5: 0x6600, 0x0ad6: 0x6600, - // Block 0x2c, offset 0xb00 - 0x0b3e: 0x6600, - // Block 0x2d, offset 0xb40 - 0x0b4a: 0x6609, - 0x0b4f: 0x6600, - 0x0b59: 0x8800, 0x0b5a: 0x1100, 0x0b5c: 0x9900, 0x0b5d: 0x1100, - 0x0b5e: 0x1100, 0x0b5f: 0x6600, - // Block 0x2e, offset 0xb80 - 0x0bb3: 0x3000, - 0x0bb8: 0x0067, 0x0bb9: 0x0067, 0x0bba: 0x0009, - // Block 0x2f, offset 0xbc0 - 0x0bc8: 0x006b, 0x0bc9: 0x006b, 0x0bca: 0x006b, 0x0bcb: 0x006b, - // Block 0x30, offset 0xc00 - 0x0c33: 0x3000, - 0x0c38: 0x0076, 0x0c39: 0x0076, - // Block 0x31, offset 0xc40 - 0x0c48: 0x007a, 0x0c49: 0x007a, 0x0c4a: 0x007a, 0x0c4b: 0x007a, - 0x0c5c: 0x3000, 0x0c5d: 0x3000, - // Block 0x32, offset 0xc80 - 0x0c8c: 0x3000, - 0x0c98: 0x00dc, 0x0c99: 0x00dc, - 0x0cb5: 0x00dc, - 0x0cb7: 0x00dc, 0x0cb9: 0x00d8, - // Block 0x33, offset 0xcc0 - 0x0cc3: 0x3300, - 0x0ccd: 0x3300, - 0x0cd2: 0x3300, 0x0cd7: 0x3300, - 0x0cdc: 0x3300, - 0x0ce9: 0x3300, - 0x0cf1: 0x0081, 0x0cf2: 0x0082, 0x0cf3: 0x3300, 0x0cf4: 0x0084, 0x0cf5: 0x3300, - 0x0cf6: 0x3300, 0x0cf7: 0x3000, 0x0cf8: 0x3300, 0x0cf9: 0x3000, 0x0cfa: 0x0082, 0x0cfb: 0x0082, - 0x0cfc: 0x0082, 0x0cfd: 0x0082, - // Block 0x34, offset 0xd00 - 0x0d00: 0x0082, 0x0d01: 0x3300, 0x0d02: 0x00e6, 0x0d03: 0x00e6, 0x0d04: 0x0009, - 0x0d06: 0x00e6, 0x0d07: 0x00e6, - 0x0d13: 0x3300, - 0x0d1d: 0x3300, - 0x0d22: 0x3300, - 0x0d27: 0x3300, - 0x0d2c: 0x3300, - 0x0d39: 0x3300, - // Block 0x35, offset 0xd40 - 0x0d46: 0x00dc, - // Block 0x36, offset 0xd80 - 0x0da5: 0x8800, 0x0da6: 0x1100, - 0x0dae: 0x6600, - 0x0db7: 0x0007, 0x0db9: 0x0009, 0x0dba: 0x0009, - // Block 0x37, offset 0xdc0 - 0x0dcd: 0x00dc, - // Block 0x38, offset 0xe00 - 0x0e3c: 0x3000, - // Block 0x39, offset 0xe40 - 0x0e61: 0x6600, 0x0e62: 0x6600, 0x0e63: 0x6600, - 0x0e64: 0x6600, 0x0e65: 0x6600, 0x0e66: 0x6600, 0x0e67: 0x6600, 0x0e68: 0x6600, 0x0e69: 0x6600, - 0x0e6a: 0x6600, 0x0e6b: 0x6600, 0x0e6c: 0x6600, 0x0e6d: 0x6600, 0x0e6e: 0x6600, 0x0e6f: 0x6600, - 0x0e70: 0x6600, 0x0e71: 0x6600, 0x0e72: 0x6600, 0x0e73: 0x6600, 0x0e74: 0x6600, 0x0e75: 0x6600, - // Block 0x3a, offset 0xe80 - 0x0ea8: 0x6600, 0x0ea9: 0x6600, - 0x0eaa: 0x6600, 0x0eab: 0x6600, 0x0eac: 0x6600, 0x0ead: 0x6600, 0x0eae: 0x6600, 0x0eaf: 0x6600, - 0x0eb0: 0x6600, 0x0eb1: 0x6600, 0x0eb2: 0x6600, 0x0eb3: 0x6600, 0x0eb4: 0x6600, 0x0eb5: 0x6600, - 0x0eb6: 0x6600, 0x0eb7: 0x6600, 0x0eb8: 0x6600, 0x0eb9: 0x6600, 0x0eba: 0x6600, 0x0ebb: 0x6600, - 0x0ebc: 0x6600, 0x0ebd: 0x6600, 0x0ebe: 0x6600, 0x0ebf: 0x6600, - // Block 0x3b, offset 0xec0 - 0x0ec0: 0x6600, 0x0ec1: 0x6600, 0x0ec2: 0x6600, - // Block 0x3c, offset 0xf00 - 0x0f1d: 0x00e6, - 0x0f1e: 0x00e6, 0x0f1f: 0x00e6, - // Block 0x3d, offset 0xf40 - 0x0f54: 0x0009, - 0x0f74: 0x0009, - // Block 0x3e, offset 0xf80 - 0x0f92: 0x0009, - 0x0f9d: 0x00e6, - // Block 0x3f, offset 0xfc0 - 0x0fe9: 0x00e4, - // Block 0x40, offset 0x1000 - 0x1039: 0x00de, 0x103a: 0x00e6, 0x103b: 0x00dc, - // Block 0x41, offset 0x1040 - 0x1057: 0x00e6, - 0x1058: 0x00dc, - // Block 0x42, offset 0x1080 - 0x10a0: 0x0009, - 0x10b5: 0x00e6, - 0x10b6: 0x00e6, 0x10b7: 0x00e6, 0x10b8: 0x00e6, 0x10b9: 0x00e6, 0x10ba: 0x00e6, 0x10bb: 0x00e6, - 0x10bc: 0x00e6, 0x10bf: 0x00dc, - // Block 0x43, offset 0x10c0 - 0x10c5: 0x8800, - 0x10c6: 0x1100, 0x10c7: 0x8800, 0x10c8: 0x1100, 0x10c9: 0x8800, 0x10ca: 0x1100, 0x10cb: 0x8800, - 0x10cc: 0x1100, 0x10cd: 0x8800, 0x10ce: 0x1100, 0x10d1: 0x8800, - 0x10d2: 0x1100, - 0x10f4: 0x0007, 0x10f5: 0x6600, - 0x10fa: 0x8800, 0x10fb: 0x1100, - 0x10fc: 0x8800, 0x10fd: 0x1100, 0x10fe: 0x8800, 0x10ff: 0x8800, - // Block 0x44, offset 0x1100 - 0x1100: 0x1100, 0x1101: 0x1100, 0x1102: 0x8800, 0x1103: 0x1100, 0x1104: 0x0009, - 0x112b: 0x00e6, 0x112c: 0x00dc, 0x112d: 0x00e6, 0x112e: 0x00e6, 0x112f: 0x00e6, - 0x1130: 0x00e6, 0x1131: 0x00e6, 0x1132: 0x00e6, 0x1133: 0x00e6, - // Block 0x45, offset 0x1140 - 0x116a: 0x0009, - // Block 0x46, offset 0x1180 - 0x11a6: 0x0007, - 0x11b2: 0x0009, 0x11b3: 0x0009, - // Block 0x47, offset 0x11c0 - 0x11f7: 0x0007, - // Block 0x48, offset 0x1200 - 0x1210: 0x00e6, 0x1211: 0x00e6, - 0x1212: 0x00e6, 0x1214: 0x0001, 0x1215: 0x00dc, 0x1216: 0x00dc, 0x1217: 0x00dc, - 0x1218: 0x00dc, 0x1219: 0x00dc, 0x121a: 0x00e6, 0x121b: 0x00e6, 0x121c: 0x00dc, 0x121d: 0x00dc, - 0x121e: 0x00dc, 0x121f: 0x00dc, 0x1220: 0x00e6, 0x1222: 0x0001, 0x1223: 0x0001, - 0x1224: 0x0001, 0x1225: 0x0001, 0x1226: 0x0001, 0x1227: 0x0001, 0x1228: 0x0001, - 0x122d: 0x00dc, - // Block 0x49, offset 0x1240 - 0x126c: 0x3000, 0x126d: 0x3000, 0x126e: 0x3000, - 0x1270: 0x3000, 0x1271: 0x3000, 0x1272: 0x3000, 0x1273: 0x3000, 0x1274: 0x3000, 0x1275: 0x3000, - 0x1276: 0x3000, 0x1277: 0x3000, 0x1278: 0x3000, 0x1279: 0x3000, 0x127a: 0x3000, - 0x127c: 0x3000, 0x127d: 0x3000, 0x127e: 0x3000, 0x127f: 0x3000, - // Block 0x4a, offset 0x1280 - 0x1280: 0x3000, 0x1281: 0x3000, 0x1282: 0x3000, 0x1283: 0x3000, 0x1284: 0x3000, 0x1285: 0x3000, - 0x1286: 0x3000, 0x1287: 0x3000, 0x1288: 0x3000, 0x1289: 0x3000, 0x128a: 0x3000, 0x128b: 0x3000, - 0x128c: 0x3000, 0x128d: 0x3000, 0x128f: 0x3000, 0x1290: 0x3000, 0x1291: 0x3000, - 0x1292: 0x3000, 0x1293: 0x3000, 0x1294: 0x3000, 0x1295: 0x3000, 0x1296: 0x3000, 0x1297: 0x3000, - 0x1298: 0x3000, 0x1299: 0x3000, 0x129a: 0x3000, 0x129b: 0x3000, 0x129c: 0x3000, 0x129d: 0x3000, - 0x129e: 0x3000, 0x129f: 0x3000, 0x12a0: 0x3000, 0x12a1: 0x3000, 0x12a2: 0x3000, 0x12a3: 0x3000, - 0x12a4: 0x3000, 0x12a5: 0x3000, 0x12a6: 0x3000, 0x12a7: 0x3000, 0x12a8: 0x3000, 0x12a9: 0x3000, - 0x12aa: 0x3000, - 0x12b8: 0x3000, - // Block 0x4b, offset 0x12c0 - 0x12db: 0x3000, 0x12dc: 0x3000, 0x12dd: 0x3000, - 0x12de: 0x3000, 0x12df: 0x3000, 0x12e0: 0x3000, 0x12e1: 0x3000, 0x12e2: 0x3000, 0x12e3: 0x3000, - 0x12e4: 0x3000, 0x12e5: 0x3000, 0x12e6: 0x3000, 0x12e7: 0x3000, 0x12e8: 0x3000, 0x12e9: 0x3000, - 0x12ea: 0x3000, 0x12eb: 0x3000, 0x12ec: 0x3000, 0x12ed: 0x3000, 0x12ee: 0x3000, 0x12ef: 0x3000, - 0x12f0: 0x3000, 0x12f1: 0x3000, 0x12f2: 0x3000, 0x12f3: 0x3000, 0x12f4: 0x3000, 0x12f5: 0x3000, - 0x12f6: 0x3000, 0x12f7: 0x3000, 0x12f8: 0x3000, 0x12f9: 0x3000, 0x12fa: 0x3000, 0x12fb: 0x3000, - 0x12fc: 0x3000, 0x12fd: 0x3000, 0x12fe: 0x3000, 0x12ff: 0x3000, - // Block 0x4c, offset 0x1300 - 0x1300: 0x00e6, 0x1301: 0x00e6, 0x1302: 0x00dc, 0x1303: 0x00e6, 0x1304: 0x00e6, 0x1305: 0x00e6, - 0x1306: 0x00e6, 0x1307: 0x00e6, 0x1308: 0x00e6, 0x1309: 0x00e6, 0x130a: 0x00dc, 0x130b: 0x00e6, - 0x130c: 0x00e6, 0x130d: 0x00ea, 0x130e: 0x00d6, 0x130f: 0x00dc, 0x1310: 0x00ca, 0x1311: 0x00e6, - 0x1312: 0x00e6, 0x1313: 0x00e6, 0x1314: 0x00e6, 0x1315: 0x00e6, 0x1316: 0x00e6, 0x1317: 0x00e6, - 0x1318: 0x00e6, 0x1319: 0x00e6, 0x131a: 0x00e6, 0x131b: 0x00e6, 0x131c: 0x00e6, 0x131d: 0x00e6, - 0x131e: 0x00e6, 0x131f: 0x00e6, 0x1320: 0x00e6, 0x1321: 0x00e6, 0x1322: 0x00e6, 0x1323: 0x00e6, - 0x1324: 0x00e6, 0x1325: 0x00e6, 0x1326: 0x00e6, - 0x133c: 0x00e9, 0x133d: 0x00dc, 0x133e: 0x00e6, 0x133f: 0x00dc, - // Block 0x4d, offset 0x1340 - 0x1340: 0x1100, 0x1341: 0x1100, 0x1342: 0x1100, 0x1343: 0x1100, 0x1344: 0x1100, 0x1345: 0x1100, - 0x1346: 0x1100, 0x1347: 0x1100, 0x1348: 0x1100, 0x1349: 0x1100, 0x134a: 0x1100, 0x134b: 0x1100, - 0x134c: 0x1100, 0x134d: 0x1100, 0x134e: 0x1100, 0x134f: 0x1100, 0x1350: 0x1100, 0x1351: 0x1100, - 0x1352: 0x1100, 0x1353: 0x1100, 0x1354: 0x1100, 0x1355: 0x1100, 0x1356: 0x1100, 0x1357: 0x1100, - 0x1358: 0x1100, 0x1359: 0x1100, 0x135a: 0x1100, 0x135b: 0x1100, 0x135c: 0x1100, 0x135d: 0x1100, - 0x135e: 0x1100, 0x135f: 0x1100, 0x1360: 0x1100, 0x1361: 0x1100, 0x1362: 0x1100, 0x1363: 0x1100, - 0x1364: 0x1100, 0x1365: 0x1100, 0x1366: 0x1100, 0x1367: 0x1100, 0x1368: 0x1100, 0x1369: 0x1100, - 0x136a: 0x1100, 0x136b: 0x1100, 0x136c: 0x1100, 0x136d: 0x1100, 0x136e: 0x1100, 0x136f: 0x1100, - 0x1370: 0x1100, 0x1371: 0x1100, 0x1372: 0x1100, 0x1373: 0x1100, 0x1374: 0x1100, 0x1375: 0x1100, - 0x1376: 0x9900, 0x1377: 0x9900, 0x1378: 0x1100, 0x1379: 0x1100, 0x137a: 0x1100, 0x137b: 0x1100, - 0x137c: 0x1100, 0x137d: 0x1100, 0x137e: 0x1100, 0x137f: 0x1100, - // Block 0x4e, offset 0x1380 - 0x1380: 0x1100, 0x1381: 0x1100, 0x1382: 0x1100, 0x1383: 0x1100, 0x1384: 0x1100, 0x1385: 0x1100, - 0x1386: 0x1100, 0x1387: 0x1100, 0x1388: 0x1100, 0x1389: 0x1100, 0x138a: 0x1100, 0x138b: 0x1100, - 0x138c: 0x1100, 0x138d: 0x1100, 0x138e: 0x1100, 0x138f: 0x1100, 0x1390: 0x1100, 0x1391: 0x1100, - 0x1392: 0x1100, 0x1393: 0x1100, 0x1394: 0x1100, 0x1395: 0x1100, 0x1396: 0x1100, 0x1397: 0x1100, - 0x1398: 0x1100, 0x1399: 0x1100, 0x139a: 0x9900, 0x139b: 0x9900, 0x139c: 0x1100, 0x139d: 0x1100, - 0x139e: 0x1100, 0x139f: 0x1100, 0x13a0: 0x1100, 0x13a1: 0x1100, 0x13a2: 0x9900, 0x13a3: 0x9900, - 0x13a4: 0x1100, 0x13a5: 0x1100, 0x13a6: 0x1100, 0x13a7: 0x1100, 0x13a8: 0x1100, 0x13a9: 0x1100, - 0x13aa: 0x1100, 0x13ab: 0x1100, 0x13ac: 0x1100, 0x13ad: 0x1100, 0x13ae: 0x1100, 0x13af: 0x1100, - 0x13b0: 0x1100, 0x13b1: 0x1100, 0x13b2: 0x1100, 0x13b3: 0x1100, 0x13b4: 0x1100, 0x13b5: 0x1100, - 0x13b6: 0x1100, 0x13b7: 0x1100, 0x13b8: 0x1100, 0x13b9: 0x1100, 0x13ba: 0x1100, 0x13bb: 0x1100, - 0x13bc: 0x1100, 0x13bd: 0x1100, 0x13be: 0x1100, 0x13bf: 0x1100, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x1100, 0x13c1: 0x1100, 0x13c2: 0x1100, 0x13c3: 0x1100, 0x13c4: 0x1100, 0x13c5: 0x1100, - 0x13c6: 0x1100, 0x13c7: 0x1100, 0x13c8: 0x1100, 0x13c9: 0x1100, 0x13ca: 0x1100, 0x13cb: 0x1100, - 0x13cc: 0x1100, 0x13cd: 0x1100, 0x13ce: 0x1100, 0x13cf: 0x1100, 0x13d0: 0x1100, 0x13d1: 0x1100, - 0x13d2: 0x1100, 0x13d3: 0x1100, 0x13d4: 0x1100, 0x13d5: 0x1100, 0x13d6: 0x1100, 0x13d7: 0x1100, - 0x13d8: 0x1100, 0x13d9: 0x1100, 0x13da: 0x3000, 0x13db: 0x3100, - 0x13e0: 0x9900, 0x13e1: 0x9900, 0x13e2: 0x1100, 0x13e3: 0x1100, - 0x13e4: 0x1100, 0x13e5: 0x1100, 0x13e6: 0x1100, 0x13e7: 0x1100, 0x13e8: 0x1100, 0x13e9: 0x1100, - 0x13ea: 0x1100, 0x13eb: 0x1100, 0x13ec: 0x1100, 0x13ed: 0x1100, 0x13ee: 0x1100, 0x13ef: 0x1100, - 0x13f0: 0x1100, 0x13f1: 0x1100, 0x13f2: 0x1100, 0x13f3: 0x1100, 0x13f4: 0x1100, 0x13f5: 0x1100, - 0x13f6: 0x1100, 0x13f7: 0x1100, 0x13f8: 0x9900, 0x13f9: 0x9900, 0x13fa: 0x1100, 0x13fb: 0x1100, - 0x13fc: 0x1100, 0x13fd: 0x1100, 0x13fe: 0x1100, 0x13ff: 0x1100, - // Block 0x50, offset 0x1400 - 0x1400: 0x1100, 0x1401: 0x1100, 0x1402: 0x1100, 0x1403: 0x1100, 0x1404: 0x1100, 0x1405: 0x1100, - 0x1406: 0x1100, 0x1407: 0x1100, 0x1408: 0x1100, 0x1409: 0x1100, 0x140a: 0x1100, 0x140b: 0x1100, - 0x140c: 0x9900, 0x140d: 0x9900, 0x140e: 0x1100, 0x140f: 0x1100, 0x1410: 0x1100, 0x1411: 0x1100, - 0x1412: 0x1100, 0x1413: 0x1100, 0x1414: 0x1100, 0x1415: 0x1100, 0x1416: 0x1100, 0x1417: 0x1100, - 0x1418: 0x1100, 0x1419: 0x1100, 0x141a: 0x1100, 0x141b: 0x1100, 0x141c: 0x1100, 0x141d: 0x1100, - 0x141e: 0x1100, 0x141f: 0x1100, 0x1420: 0x1100, 0x1421: 0x1100, 0x1422: 0x1100, 0x1423: 0x1100, - 0x1424: 0x1100, 0x1425: 0x1100, 0x1426: 0x1100, 0x1427: 0x1100, 0x1428: 0x1100, 0x1429: 0x1100, - 0x142a: 0x1100, 0x142b: 0x1100, 0x142c: 0x1100, 0x142d: 0x1100, 0x142e: 0x1100, 0x142f: 0x1100, - 0x1430: 0x1100, 0x1431: 0x1100, 0x1432: 0x1100, 0x1433: 0x1100, 0x1434: 0x1100, 0x1435: 0x1100, - 0x1436: 0x1100, 0x1437: 0x1100, 0x1438: 0x1100, 0x1439: 0x1100, - // Block 0x51, offset 0x1440 - 0x1440: 0x9900, 0x1441: 0x9900, 0x1442: 0x9900, 0x1443: 0x9900, 0x1444: 0x9900, 0x1445: 0x9900, - 0x1446: 0x9900, 0x1447: 0x9900, 0x1448: 0x9900, 0x1449: 0x9900, 0x144a: 0x9900, 0x144b: 0x9900, - 0x144c: 0x9900, 0x144d: 0x9900, 0x144e: 0x9900, 0x144f: 0x9900, 0x1450: 0x9900, 0x1451: 0x9900, - 0x1452: 0x1100, 0x1453: 0x1100, 0x1454: 0x1100, 0x1455: 0x1100, - 0x1458: 0x9900, 0x1459: 0x9900, 0x145a: 0x1100, 0x145b: 0x1100, 0x145c: 0x1100, 0x145d: 0x1100, - 0x1460: 0x9900, 0x1461: 0x9900, 0x1462: 0x9900, 0x1463: 0x9900, - 0x1464: 0x9900, 0x1465: 0x9900, 0x1466: 0x9900, 0x1467: 0x9900, 0x1468: 0x9900, 0x1469: 0x9900, - 0x146a: 0x9900, 0x146b: 0x9900, 0x146c: 0x9900, 0x146d: 0x9900, 0x146e: 0x9900, 0x146f: 0x9900, - 0x1470: 0x9900, 0x1471: 0x9900, 0x1472: 0x1100, 0x1473: 0x1100, 0x1474: 0x1100, 0x1475: 0x1100, - 0x1476: 0x1100, 0x1477: 0x1100, 0x1478: 0x9900, 0x1479: 0x9900, 0x147a: 0x1100, 0x147b: 0x1100, - 0x147c: 0x1100, 0x147d: 0x1100, 0x147e: 0x1100, 0x147f: 0x1100, - // Block 0x52, offset 0x1480 - 0x1480: 0x9900, 0x1481: 0x9900, 0x1482: 0x1100, 0x1483: 0x1100, 0x1484: 0x1100, 0x1485: 0x1100, - 0x1488: 0x9900, 0x1489: 0x9900, 0x148a: 0x1100, 0x148b: 0x1100, - 0x148c: 0x1100, 0x148d: 0x1100, 0x1490: 0x9900, 0x1491: 0x9900, - 0x1492: 0x1100, 0x1493: 0x1100, 0x1494: 0x1100, 0x1495: 0x1100, 0x1496: 0x1100, 0x1497: 0x1100, - 0x1499: 0x9900, 0x149b: 0x1100, 0x149d: 0x1100, - 0x149f: 0x1100, 0x14a0: 0x9900, 0x14a1: 0x9900, 0x14a2: 0x9900, 0x14a3: 0x9900, - 0x14a4: 0x9900, 0x14a5: 0x9900, 0x14a6: 0x9900, 0x14a7: 0x9900, 0x14a8: 0x9900, 0x14a9: 0x9900, - 0x14aa: 0x9900, 0x14ab: 0x9900, 0x14ac: 0x9900, 0x14ad: 0x9900, 0x14ae: 0x9900, 0x14af: 0x9900, - 0x14b0: 0x9900, 0x14b1: 0x3300, 0x14b2: 0x1100, 0x14b3: 0x3300, 0x14b4: 0x9900, 0x14b5: 0x3300, - 0x14b6: 0x1100, 0x14b7: 0x3300, 0x14b8: 0x1100, 0x14b9: 0x3300, 0x14ba: 0x1100, 0x14bb: 0x3300, - 0x14bc: 0x9900, 0x14bd: 0x3300, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x1100, 0x14c1: 0x1100, 0x14c2: 0x1100, 0x14c3: 0x1100, 0x14c4: 0x1100, 0x14c5: 0x1100, - 0x14c6: 0x1100, 0x14c7: 0x1100, 0x14c8: 0x1100, 0x14c9: 0x1100, 0x14ca: 0x1100, 0x14cb: 0x1100, - 0x14cc: 0x1100, 0x14cd: 0x1100, 0x14ce: 0x1100, 0x14cf: 0x1100, 0x14d0: 0x1100, 0x14d1: 0x1100, - 0x14d2: 0x1100, 0x14d3: 0x1100, 0x14d4: 0x1100, 0x14d5: 0x1100, 0x14d6: 0x1100, 0x14d7: 0x1100, - 0x14d8: 0x1100, 0x14d9: 0x1100, 0x14da: 0x1100, 0x14db: 0x1100, 0x14dc: 0x1100, 0x14dd: 0x1100, - 0x14de: 0x1100, 0x14df: 0x1100, 0x14e0: 0x1100, 0x14e1: 0x1100, 0x14e2: 0x1100, 0x14e3: 0x1100, - 0x14e4: 0x1100, 0x14e5: 0x1100, 0x14e6: 0x1100, 0x14e7: 0x1100, 0x14e8: 0x1100, 0x14e9: 0x1100, - 0x14ea: 0x1100, 0x14eb: 0x1100, 0x14ec: 0x1100, 0x14ed: 0x1100, 0x14ee: 0x1100, 0x14ef: 0x1100, - 0x14f0: 0x1100, 0x14f1: 0x1100, 0x14f2: 0x1100, 0x14f3: 0x1100, 0x14f4: 0x1100, - 0x14f6: 0x9900, 0x14f7: 0x1100, 0x14f8: 0x1100, 0x14f9: 0x1100, 0x14fa: 0x1100, 0x14fb: 0x3300, - 0x14fc: 0x1100, 0x14fd: 0x3000, 0x14fe: 0x3300, 0x14ff: 0x3800, - // Block 0x54, offset 0x1500 - 0x1500: 0x3000, 0x1501: 0x3100, 0x1502: 0x1100, 0x1503: 0x1100, 0x1504: 0x1100, - 0x1506: 0x9900, 0x1507: 0x1100, 0x1508: 0x1100, 0x1509: 0x3300, 0x150a: 0x1100, 0x150b: 0x3300, - 0x150c: 0x1100, 0x150d: 0x3100, 0x150e: 0x3100, 0x150f: 0x3100, 0x1510: 0x1100, 0x1511: 0x1100, - 0x1512: 0x1100, 0x1513: 0x3300, 0x1516: 0x1100, 0x1517: 0x1100, - 0x1518: 0x1100, 0x1519: 0x1100, 0x151a: 0x1100, 0x151b: 0x3300, 0x151d: 0x3100, - 0x151e: 0x3100, 0x151f: 0x3100, 0x1520: 0x1100, 0x1521: 0x1100, 0x1522: 0x1100, 0x1523: 0x3300, - 0x1524: 0x1100, 0x1525: 0x1100, 0x1526: 0x1100, 0x1527: 0x1100, 0x1528: 0x1100, 0x1529: 0x1100, - 0x152a: 0x1100, 0x152b: 0x3300, 0x152c: 0x1100, 0x152d: 0x3100, 0x152e: 0x3300, 0x152f: 0x3300, - 0x1532: 0x1100, 0x1533: 0x1100, 0x1534: 0x1100, - 0x1536: 0x9900, 0x1537: 0x1100, 0x1538: 0x1100, 0x1539: 0x3300, 0x153a: 0x1100, 0x153b: 0x3300, - 0x153c: 0x1100, 0x153d: 0x3300, 0x153e: 0x3800, - // Block 0x55, offset 0x1540 - 0x1540: 0x3300, 0x1541: 0x3300, 0x1542: 0x3000, 0x1543: 0x3000, 0x1544: 0x3000, 0x1545: 0x3000, - 0x1546: 0x3000, 0x1547: 0x3000, 0x1548: 0x3000, 0x1549: 0x3000, 0x154a: 0x3000, - 0x1551: 0x3000, - 0x1557: 0x3000, - 0x1564: 0x3000, 0x1565: 0x3000, 0x1566: 0x3000, - 0x156f: 0x3000, - 0x1573: 0x3000, 0x1574: 0x3000, - 0x1576: 0x3000, 0x1577: 0x3000, - 0x157c: 0x3000, 0x157e: 0x3000, - // Block 0x56, offset 0x1580 - 0x1587: 0x3000, 0x1588: 0x3000, 0x1589: 0x3000, - 0x1597: 0x3000, - 0x159f: 0x3000, - 0x15b0: 0x3000, 0x15b1: 0x3000, 0x15b4: 0x3000, 0x15b5: 0x3000, - 0x15b6: 0x3000, 0x15b7: 0x3000, 0x15b8: 0x3000, 0x15b9: 0x3000, 0x15ba: 0x3000, 0x15bb: 0x3000, - 0x15bc: 0x3000, 0x15bd: 0x3000, 0x15be: 0x3000, 0x15bf: 0x3000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x3000, 0x15c1: 0x3000, 0x15c2: 0x3000, 0x15c3: 0x3000, 0x15c4: 0x3000, 0x15c5: 0x3000, - 0x15c6: 0x3000, 0x15c7: 0x3000, 0x15c8: 0x3000, 0x15c9: 0x3000, 0x15ca: 0x3000, 0x15cb: 0x3000, - 0x15cc: 0x3000, 0x15cd: 0x3000, 0x15ce: 0x3000, 0x15d0: 0x3000, 0x15d1: 0x3000, - 0x15d2: 0x3000, 0x15d3: 0x3000, 0x15d4: 0x3000, 0x15d5: 0x3000, 0x15d6: 0x3000, 0x15d7: 0x3000, - 0x15d8: 0x3000, 0x15d9: 0x3000, 0x15da: 0x3000, 0x15db: 0x3000, 0x15dc: 0x3000, - 0x15e8: 0x3000, - // Block 0x58, offset 0x1600 - 0x1610: 0x00e6, 0x1611: 0x00e6, - 0x1612: 0x0001, 0x1613: 0x0001, 0x1614: 0x00e6, 0x1615: 0x00e6, 0x1616: 0x00e6, 0x1617: 0x00e6, - 0x1618: 0x0001, 0x1619: 0x0001, 0x161a: 0x0001, 0x161b: 0x00e6, 0x161c: 0x00e6, - 0x1621: 0x00e6, - 0x1625: 0x0001, 0x1626: 0x0001, 0x1627: 0x00e6, 0x1628: 0x00dc, 0x1629: 0x00e6, - 0x162a: 0x0001, 0x162b: 0x0001, 0x162c: 0x00dc, 0x162d: 0x00dc, 0x162e: 0x00dc, 0x162f: 0x00dc, - 0x1630: 0x00e6, - // Block 0x59, offset 0x1640 - 0x1640: 0x3000, 0x1641: 0x3000, 0x1642: 0x3000, 0x1643: 0x3000, 0x1645: 0x3000, - 0x1646: 0x3000, 0x1647: 0x3000, 0x1649: 0x3000, 0x164a: 0x3000, 0x164b: 0x3000, - 0x164c: 0x3000, 0x164d: 0x3000, 0x164e: 0x3000, 0x164f: 0x3000, 0x1650: 0x3000, 0x1651: 0x3000, - 0x1652: 0x3000, 0x1653: 0x3000, 0x1655: 0x3000, 0x1656: 0x3000, - 0x1659: 0x3000, 0x165a: 0x3000, 0x165b: 0x3000, 0x165c: 0x3000, 0x165d: 0x3000, - 0x1660: 0x3000, 0x1661: 0x3000, 0x1662: 0x3000, - 0x1664: 0x3000, 0x1666: 0x3300, 0x1668: 0x3000, - 0x166a: 0x3300, 0x166b: 0x3300, 0x166c: 0x3000, 0x166d: 0x3000, 0x166f: 0x3000, - 0x1670: 0x3000, 0x1671: 0x3000, 0x1673: 0x3000, 0x1674: 0x3000, 0x1675: 0x3000, - 0x1676: 0x3000, 0x1677: 0x3000, 0x1678: 0x3000, 0x1679: 0x3000, 0x167b: 0x3000, - 0x167c: 0x3000, 0x167d: 0x3000, 0x167e: 0x3000, 0x167f: 0x3000, - // Block 0x5a, offset 0x1680 - 0x1680: 0x3000, 0x1685: 0x3000, - 0x1686: 0x3000, 0x1687: 0x3000, 0x1688: 0x3000, 0x1689: 0x3000, - 0x1690: 0x3000, 0x1691: 0x3000, - 0x1692: 0x3000, 0x1693: 0x3000, 0x1694: 0x3000, 0x1695: 0x3000, 0x1696: 0x3000, 0x1697: 0x3000, - 0x1698: 0x3000, 0x1699: 0x3000, 0x169a: 0x3000, 0x169b: 0x3000, 0x169c: 0x3000, 0x169d: 0x3000, - 0x169e: 0x3000, 0x169f: 0x3000, 0x16a0: 0x3000, 0x16a1: 0x3000, 0x16a2: 0x3000, 0x16a3: 0x3000, - 0x16a4: 0x3000, 0x16a5: 0x3000, 0x16a6: 0x3000, 0x16a7: 0x3000, 0x16a8: 0x3000, 0x16a9: 0x3000, - 0x16aa: 0x3000, 0x16ab: 0x3000, 0x16ac: 0x3000, 0x16ad: 0x3000, 0x16ae: 0x3000, 0x16af: 0x3000, - 0x16b0: 0x3000, 0x16b1: 0x3000, 0x16b2: 0x3000, 0x16b3: 0x3000, 0x16b4: 0x3000, 0x16b5: 0x3000, - 0x16b6: 0x3000, 0x16b7: 0x3000, 0x16b8: 0x3000, 0x16b9: 0x3000, 0x16ba: 0x3000, 0x16bb: 0x3000, - 0x16bc: 0x3000, 0x16bd: 0x3000, 0x16be: 0x3000, 0x16bf: 0x3000, - // Block 0x5b, offset 0x16c0 - 0x16c9: 0x3000, - 0x16d0: 0x8800, - 0x16d2: 0x8800, 0x16d4: 0x8800, - 0x16da: 0x1100, 0x16db: 0x1100, - 0x16ee: 0x1100, - // Block 0x5c, offset 0x1700 - 0x170d: 0x1100, 0x170e: 0x1100, 0x170f: 0x1100, 0x1710: 0x8800, - 0x1712: 0x8800, 0x1714: 0x8800, - // Block 0x5d, offset 0x1740 - 0x1743: 0x8800, 0x1744: 0x1100, - 0x1748: 0x8800, 0x1749: 0x1100, 0x174b: 0x8800, - 0x174c: 0x1100, - 0x1763: 0x8800, - 0x1764: 0x1100, 0x1765: 0x8800, 0x1766: 0x1100, - 0x176c: 0x3000, 0x176d: 0x3000, 0x176f: 0x3000, - 0x1770: 0x3000, - 0x177c: 0x8800, - // Block 0x5e, offset 0x1780 - 0x1781: 0x1100, 0x1783: 0x8800, 0x1784: 0x1100, 0x1785: 0x8800, - 0x1787: 0x1100, 0x1788: 0x8800, 0x1789: 0x1100, - 0x178d: 0x8800, - 0x17a0: 0x1100, 0x17a1: 0x8800, 0x17a2: 0x1100, - 0x17a4: 0x8800, 0x17a5: 0x8800, - 0x17ad: 0x1100, 0x17ae: 0x1100, 0x17af: 0x1100, - 0x17b0: 0x1100, 0x17b1: 0x1100, 0x17b2: 0x8800, 0x17b3: 0x8800, 0x17b4: 0x1100, 0x17b5: 0x1100, - 0x17b6: 0x8800, 0x17b7: 0x8800, 0x17b8: 0x1100, 0x17b9: 0x1100, 0x17ba: 0x8800, 0x17bb: 0x8800, - 0x17bc: 0x8800, 0x17bd: 0x8800, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x1100, 0x17c1: 0x1100, 0x17c2: 0x8800, 0x17c3: 0x8800, 0x17c4: 0x1100, 0x17c5: 0x1100, - 0x17c6: 0x8800, 0x17c7: 0x8800, 0x17c8: 0x1100, 0x17c9: 0x1100, - 0x17d1: 0x8800, - 0x17d2: 0x8800, - 0x17e2: 0x8800, - 0x17e8: 0x8800, 0x17e9: 0x8800, - 0x17eb: 0x8800, 0x17ec: 0x1100, 0x17ed: 0x1100, 0x17ee: 0x1100, 0x17ef: 0x1100, - 0x17f2: 0x8800, 0x17f3: 0x8800, 0x17f4: 0x8800, 0x17f5: 0x8800, - // Block 0x60, offset 0x1800 - 0x1820: 0x1100, 0x1821: 0x1100, 0x1822: 0x1100, 0x1823: 0x1100, - 0x182a: 0x1100, 0x182b: 0x1100, 0x182c: 0x1100, 0x182d: 0x1100, - // Block 0x61, offset 0x1840 - 0x1869: 0x3300, - 0x186a: 0x3300, - // Block 0x62, offset 0x1880 - 0x18a0: 0x3000, 0x18a1: 0x3000, 0x18a2: 0x3000, 0x18a3: 0x3000, - 0x18a4: 0x3000, 0x18a5: 0x3000, 0x18a6: 0x3000, 0x18a7: 0x3000, 0x18a8: 0x3000, 0x18a9: 0x3000, - 0x18aa: 0x3000, 0x18ab: 0x3000, 0x18ac: 0x3000, 0x18ad: 0x3000, 0x18ae: 0x3000, 0x18af: 0x3000, - 0x18b0: 0x3000, 0x18b1: 0x3000, 0x18b2: 0x3000, 0x18b3: 0x3000, 0x18b4: 0x3000, 0x18b5: 0x3000, - 0x18b6: 0x3000, 0x18b7: 0x3000, 0x18b8: 0x3000, 0x18b9: 0x3000, 0x18ba: 0x3000, 0x18bb: 0x3000, - 0x18bc: 0x3000, 0x18bd: 0x3000, 0x18be: 0x3000, 0x18bf: 0x3000, - // Block 0x63, offset 0x18c0 - 0x18c0: 0x3000, 0x18c1: 0x3000, 0x18c2: 0x3000, 0x18c3: 0x3000, 0x18c4: 0x3000, 0x18c5: 0x3000, - 0x18c6: 0x3000, 0x18c7: 0x3000, 0x18c8: 0x3000, 0x18c9: 0x3000, 0x18ca: 0x3000, 0x18cb: 0x3000, - 0x18cc: 0x3000, 0x18cd: 0x3000, 0x18ce: 0x3000, 0x18cf: 0x3000, 0x18d0: 0x3000, 0x18d1: 0x3000, - 0x18d2: 0x3000, 0x18d3: 0x3000, 0x18d4: 0x3000, 0x18d5: 0x3000, 0x18d6: 0x3000, 0x18d7: 0x3000, - 0x18d8: 0x3000, 0x18d9: 0x3000, 0x18da: 0x3000, 0x18db: 0x3000, 0x18dc: 0x3000, 0x18dd: 0x3000, - 0x18de: 0x3000, 0x18df: 0x3000, 0x18e0: 0x3000, 0x18e1: 0x3000, 0x18e2: 0x3000, 0x18e3: 0x3000, - 0x18e4: 0x3000, 0x18e5: 0x3000, 0x18e6: 0x3000, 0x18e7: 0x3000, 0x18e8: 0x3000, 0x18e9: 0x3000, - 0x18ea: 0x3000, 0x18eb: 0x3000, 0x18ec: 0x3000, 0x18ed: 0x3000, 0x18ee: 0x3000, 0x18ef: 0x3000, - 0x18f0: 0x3000, 0x18f1: 0x3000, 0x18f2: 0x3000, 0x18f3: 0x3000, 0x18f4: 0x3000, 0x18f5: 0x3000, - 0x18f6: 0x3000, 0x18f7: 0x3000, 0x18f8: 0x3000, 0x18f9: 0x3000, 0x18fa: 0x3000, 0x18fb: 0x3000, - 0x18fc: 0x3000, 0x18fd: 0x3000, 0x18fe: 0x3000, 0x18ff: 0x3000, - // Block 0x64, offset 0x1900 - 0x1900: 0x3000, 0x1901: 0x3000, 0x1902: 0x3000, 0x1903: 0x3000, 0x1904: 0x3000, 0x1905: 0x3000, - 0x1906: 0x3000, 0x1907: 0x3000, 0x1908: 0x3000, 0x1909: 0x3000, 0x190a: 0x3000, 0x190b: 0x3000, - 0x190c: 0x3000, 0x190d: 0x3000, 0x190e: 0x3000, 0x190f: 0x3000, 0x1910: 0x3000, 0x1911: 0x3000, - 0x1912: 0x3000, 0x1913: 0x3000, 0x1914: 0x3000, 0x1915: 0x3000, 0x1916: 0x3000, 0x1917: 0x3000, - 0x1918: 0x3000, 0x1919: 0x3000, 0x191a: 0x3000, 0x191b: 0x3000, 0x191c: 0x3000, 0x191d: 0x3000, - 0x191e: 0x3000, 0x191f: 0x3000, 0x1920: 0x3000, 0x1921: 0x3000, 0x1922: 0x3000, 0x1923: 0x3000, - 0x1924: 0x3000, 0x1925: 0x3000, 0x1926: 0x3000, 0x1927: 0x3000, 0x1928: 0x3000, 0x1929: 0x3000, - 0x192a: 0x3000, - // Block 0x65, offset 0x1940 - 0x194c: 0x3000, - // Block 0x66, offset 0x1980 - 0x19b4: 0x3000, 0x19b5: 0x3000, - 0x19b6: 0x3000, - // Block 0x67, offset 0x19c0 - 0x19dc: 0x3300, - // Block 0x68, offset 0x1a00 - 0x1a3c: 0x3000, 0x1a3d: 0x3000, - // Block 0x69, offset 0x1a40 - 0x1a6f: 0x00e6, - 0x1a70: 0x00e6, 0x1a71: 0x00e6, - // Block 0x6a, offset 0x1a80 - 0x1aaf: 0x3000, - 0x1abf: 0x0009, - // Block 0x6b, offset 0x1ac0 - 0x1ae0: 0x00e6, 0x1ae1: 0x00e6, 0x1ae2: 0x00e6, 0x1ae3: 0x00e6, - 0x1ae4: 0x00e6, 0x1ae5: 0x00e6, 0x1ae6: 0x00e6, 0x1ae7: 0x00e6, 0x1ae8: 0x00e6, 0x1ae9: 0x00e6, - 0x1aea: 0x00e6, 0x1aeb: 0x00e6, 0x1aec: 0x00e6, 0x1aed: 0x00e6, 0x1aee: 0x00e6, 0x1aef: 0x00e6, - 0x1af0: 0x00e6, 0x1af1: 0x00e6, 0x1af2: 0x00e6, 0x1af3: 0x00e6, 0x1af4: 0x00e6, 0x1af5: 0x00e6, - 0x1af6: 0x00e6, 0x1af7: 0x00e6, 0x1af8: 0x00e6, 0x1af9: 0x00e6, 0x1afa: 0x00e6, 0x1afb: 0x00e6, - 0x1afc: 0x00e6, 0x1afd: 0x00e6, 0x1afe: 0x00e6, 0x1aff: 0x00e6, - // Block 0x6c, offset 0x1b00 - 0x1b1f: 0x3000, - // Block 0x6d, offset 0x1b40 - 0x1b73: 0x3000, - // Block 0x6e, offset 0x1b80 - 0x1b80: 0x3000, 0x1b81: 0x3000, 0x1b82: 0x3000, 0x1b83: 0x3000, 0x1b84: 0x3000, 0x1b85: 0x3000, - 0x1b86: 0x3000, 0x1b87: 0x3000, 0x1b88: 0x3000, 0x1b89: 0x3000, 0x1b8a: 0x3000, 0x1b8b: 0x3000, - 0x1b8c: 0x3000, 0x1b8d: 0x3000, 0x1b8e: 0x3000, 0x1b8f: 0x3000, 0x1b90: 0x3000, 0x1b91: 0x3000, - 0x1b92: 0x3000, 0x1b93: 0x3000, 0x1b94: 0x3000, 0x1b95: 0x3000, - // Block 0x6f, offset 0x1bc0 - 0x1bc0: 0x3000, - 0x1bea: 0x00da, 0x1beb: 0x00e4, 0x1bec: 0x00e8, 0x1bed: 0x00de, 0x1bee: 0x00e0, 0x1bef: 0x00e0, - 0x1bf6: 0x3000, 0x1bf8: 0x3000, 0x1bf9: 0x3000, 0x1bfa: 0x3000, - // Block 0x70, offset 0x1c00 - 0x1c06: 0x8800, 0x1c0b: 0x8800, - 0x1c0c: 0x1100, 0x1c0d: 0x8800, 0x1c0e: 0x1100, 0x1c0f: 0x8800, 0x1c10: 0x1100, 0x1c11: 0x8800, - 0x1c12: 0x1100, 0x1c13: 0x8800, 0x1c14: 0x1100, 0x1c15: 0x8800, 0x1c16: 0x1100, 0x1c17: 0x8800, - 0x1c18: 0x1100, 0x1c19: 0x8800, 0x1c1a: 0x1100, 0x1c1b: 0x8800, 0x1c1c: 0x1100, 0x1c1d: 0x8800, - 0x1c1e: 0x1100, 0x1c1f: 0x8800, 0x1c20: 0x1100, 0x1c21: 0x8800, 0x1c22: 0x1100, - 0x1c24: 0x8800, 0x1c25: 0x1100, 0x1c26: 0x8800, 0x1c27: 0x1100, 0x1c28: 0x8800, 0x1c29: 0x1100, - 0x1c2f: 0x8800, - 0x1c30: 0x1100, 0x1c31: 0x1100, 0x1c32: 0x8800, 0x1c33: 0x1100, 0x1c34: 0x1100, 0x1c35: 0x8800, - 0x1c36: 0x1100, 0x1c37: 0x1100, 0x1c38: 0x8800, 0x1c39: 0x1100, 0x1c3a: 0x1100, 0x1c3b: 0x8800, - 0x1c3c: 0x1100, 0x1c3d: 0x1100, - // Block 0x71, offset 0x1c40 - 0x1c54: 0x1100, - 0x1c59: 0x6608, 0x1c5a: 0x6608, 0x1c5b: 0x3000, 0x1c5c: 0x3000, 0x1c5d: 0x8800, - 0x1c5e: 0x1100, 0x1c5f: 0x3000, - 0x1c66: 0x8800, - 0x1c6b: 0x8800, 0x1c6c: 0x1100, 0x1c6d: 0x8800, 0x1c6e: 0x1100, 0x1c6f: 0x8800, - 0x1c70: 0x1100, 0x1c71: 0x8800, 0x1c72: 0x1100, 0x1c73: 0x8800, 0x1c74: 0x1100, 0x1c75: 0x8800, - 0x1c76: 0x1100, 0x1c77: 0x8800, 0x1c78: 0x1100, 0x1c79: 0x8800, 0x1c7a: 0x1100, 0x1c7b: 0x8800, - 0x1c7c: 0x1100, 0x1c7d: 0x8800, 0x1c7e: 0x1100, 0x1c7f: 0x8800, - // Block 0x72, offset 0x1c80 - 0x1c80: 0x1100, 0x1c81: 0x8800, 0x1c82: 0x1100, 0x1c84: 0x8800, 0x1c85: 0x1100, - 0x1c86: 0x8800, 0x1c87: 0x1100, 0x1c88: 0x8800, 0x1c89: 0x1100, - 0x1c8f: 0x8800, 0x1c90: 0x1100, 0x1c91: 0x1100, - 0x1c92: 0x8800, 0x1c93: 0x1100, 0x1c94: 0x1100, 0x1c95: 0x8800, 0x1c96: 0x1100, 0x1c97: 0x1100, - 0x1c98: 0x8800, 0x1c99: 0x1100, 0x1c9a: 0x1100, 0x1c9b: 0x8800, 0x1c9c: 0x1100, 0x1c9d: 0x1100, - 0x1caf: 0x8800, - 0x1cb0: 0x8800, 0x1cb1: 0x8800, 0x1cb2: 0x8800, 0x1cb4: 0x1100, - 0x1cb7: 0x1100, 0x1cb8: 0x1100, 0x1cb9: 0x1100, 0x1cba: 0x1100, - 0x1cbd: 0x8800, 0x1cbe: 0x1100, 0x1cbf: 0x3000, - // Block 0x73, offset 0x1cc0 - 0x1cf1: 0x3000, 0x1cf2: 0x3000, 0x1cf3: 0x3000, 0x1cf4: 0x3000, 0x1cf5: 0x3000, - 0x1cf6: 0x3000, 0x1cf7: 0x3000, 0x1cf8: 0x3000, 0x1cf9: 0x3000, 0x1cfa: 0x3000, 0x1cfb: 0x3000, - 0x1cfc: 0x3000, 0x1cfd: 0x3000, 0x1cfe: 0x3000, 0x1cff: 0x3000, - // Block 0x74, offset 0x1d00 - 0x1d00: 0x3000, 0x1d01: 0x3000, 0x1d02: 0x3000, 0x1d03: 0x3000, 0x1d04: 0x3000, 0x1d05: 0x3000, - 0x1d06: 0x3000, 0x1d07: 0x3000, 0x1d08: 0x3000, 0x1d09: 0x3000, 0x1d0a: 0x3000, 0x1d0b: 0x3000, - 0x1d0c: 0x3000, 0x1d0d: 0x3000, 0x1d0e: 0x3000, - 0x1d12: 0x3000, 0x1d13: 0x3000, 0x1d14: 0x3000, 0x1d15: 0x3000, 0x1d16: 0x3000, 0x1d17: 0x3000, - 0x1d18: 0x3000, 0x1d19: 0x3000, 0x1d1a: 0x3000, 0x1d1b: 0x3000, 0x1d1c: 0x3000, 0x1d1d: 0x3000, - 0x1d1e: 0x3000, 0x1d1f: 0x3000, - // Block 0x75, offset 0x1d40 - 0x1d40: 0x3000, 0x1d41: 0x3000, 0x1d42: 0x3000, 0x1d43: 0x3000, 0x1d44: 0x3000, 0x1d45: 0x3000, - 0x1d46: 0x3000, 0x1d47: 0x3000, 0x1d48: 0x3000, 0x1d49: 0x3000, 0x1d4a: 0x3000, 0x1d4b: 0x3000, - 0x1d4c: 0x3000, 0x1d4d: 0x3000, 0x1d4e: 0x3000, 0x1d4f: 0x3000, 0x1d50: 0x3000, 0x1d51: 0x3000, - 0x1d52: 0x3000, 0x1d53: 0x3000, 0x1d54: 0x3000, 0x1d55: 0x3000, 0x1d56: 0x3000, 0x1d57: 0x3000, - 0x1d58: 0x3000, 0x1d59: 0x3000, 0x1d5a: 0x3000, 0x1d5b: 0x3000, 0x1d5c: 0x3000, 0x1d5d: 0x3000, - 0x1d5e: 0x3000, 0x1d60: 0x3000, 0x1d61: 0x3000, 0x1d62: 0x3000, 0x1d63: 0x3000, - 0x1d64: 0x3000, 0x1d65: 0x3000, 0x1d66: 0x3000, 0x1d67: 0x3000, 0x1d68: 0x3000, 0x1d69: 0x3000, - 0x1d6a: 0x3000, 0x1d6b: 0x3000, 0x1d6c: 0x3000, 0x1d6d: 0x3000, 0x1d6e: 0x3000, 0x1d6f: 0x3000, - 0x1d70: 0x3000, 0x1d71: 0x3000, 0x1d72: 0x3000, 0x1d73: 0x3000, 0x1d74: 0x3000, 0x1d75: 0x3000, - 0x1d76: 0x3000, 0x1d77: 0x3000, 0x1d78: 0x3000, 0x1d79: 0x3000, 0x1d7a: 0x3000, 0x1d7b: 0x3000, - 0x1d7c: 0x3000, 0x1d7d: 0x3000, 0x1d7e: 0x3000, 0x1d7f: 0x3000, - // Block 0x76, offset 0x1d80 - 0x1d80: 0x3000, 0x1d81: 0x3000, 0x1d82: 0x3000, 0x1d83: 0x3000, 0x1d84: 0x3000, 0x1d85: 0x3000, - 0x1d86: 0x3000, 0x1d87: 0x3000, - 0x1d90: 0x3000, 0x1d91: 0x3000, - 0x1d92: 0x3000, 0x1d93: 0x3000, 0x1d94: 0x3000, 0x1d95: 0x3000, 0x1d96: 0x3000, 0x1d97: 0x3000, - 0x1d98: 0x3000, 0x1d99: 0x3000, 0x1d9a: 0x3000, 0x1d9b: 0x3000, 0x1d9c: 0x3000, 0x1d9d: 0x3000, - 0x1d9e: 0x3000, 0x1d9f: 0x3000, 0x1da0: 0x3000, 0x1da1: 0x3000, 0x1da2: 0x3000, 0x1da3: 0x3000, - 0x1da4: 0x3000, 0x1da5: 0x3000, 0x1da6: 0x3000, 0x1da7: 0x3000, 0x1da8: 0x3000, 0x1da9: 0x3000, - 0x1daa: 0x3000, 0x1dab: 0x3000, 0x1dac: 0x3000, 0x1dad: 0x3000, 0x1dae: 0x3000, 0x1daf: 0x3000, - 0x1db0: 0x3000, 0x1db1: 0x3000, 0x1db2: 0x3000, 0x1db3: 0x3000, 0x1db4: 0x3000, 0x1db5: 0x3000, - 0x1db6: 0x3000, 0x1db7: 0x3000, 0x1db8: 0x3000, 0x1db9: 0x3000, 0x1dba: 0x3000, 0x1dbb: 0x3000, - 0x1dbc: 0x3000, 0x1dbd: 0x3000, 0x1dbe: 0x3000, - // Block 0x77, offset 0x1dc0 - 0x1dc0: 0x3000, 0x1dc1: 0x3000, 0x1dc2: 0x3000, 0x1dc3: 0x3000, 0x1dc4: 0x3000, 0x1dc5: 0x3000, - 0x1dc6: 0x3000, 0x1dc7: 0x3000, 0x1dc8: 0x3000, 0x1dc9: 0x3000, 0x1dca: 0x3000, 0x1dcb: 0x3000, - 0x1dcc: 0x3000, 0x1dcd: 0x3000, 0x1dce: 0x3000, 0x1dcf: 0x3000, 0x1dd0: 0x3000, 0x1dd1: 0x3000, - 0x1dd2: 0x3000, 0x1dd3: 0x3000, 0x1dd4: 0x3000, 0x1dd5: 0x3000, 0x1dd6: 0x3000, 0x1dd7: 0x3000, - 0x1dd8: 0x3000, 0x1dd9: 0x3000, 0x1dda: 0x3000, 0x1ddb: 0x3000, 0x1ddc: 0x3000, 0x1ddd: 0x3000, - 0x1dde: 0x3000, 0x1ddf: 0x3000, 0x1de0: 0x3000, 0x1de1: 0x3000, 0x1de2: 0x3000, 0x1de3: 0x3000, - 0x1de4: 0x3000, 0x1de5: 0x3000, 0x1de6: 0x3000, 0x1de7: 0x3000, 0x1de8: 0x3000, 0x1de9: 0x3000, - 0x1dea: 0x3000, 0x1deb: 0x3000, 0x1dec: 0x3000, 0x1ded: 0x3000, 0x1dee: 0x3000, 0x1def: 0x3000, - 0x1df0: 0x3000, 0x1df1: 0x3000, 0x1df2: 0x3000, 0x1df3: 0x3000, 0x1df4: 0x3000, 0x1df5: 0x3000, - 0x1df6: 0x3000, 0x1df7: 0x3000, 0x1df8: 0x3000, 0x1df9: 0x3000, 0x1dfa: 0x3000, 0x1dfb: 0x3000, - 0x1dfc: 0x3000, 0x1dfd: 0x3000, 0x1dfe: 0x3000, - // Block 0x78, offset 0x1e00 - 0x1e2f: 0x00e6, - 0x1e3c: 0x00e6, 0x1e3d: 0x00e6, - // Block 0x79, offset 0x1e40 - 0x1e70: 0x00e6, 0x1e71: 0x00e6, - // Block 0x7a, offset 0x1e80 - 0x1eb0: 0x3000, - // Block 0x7b, offset 0x1ec0 - 0x1ec6: 0x0009, - // Block 0x7c, offset 0x1f00 - 0x1f04: 0x0009, - 0x1f20: 0x00e6, 0x1f21: 0x00e6, 0x1f22: 0x00e6, 0x1f23: 0x00e6, - 0x1f24: 0x00e6, 0x1f25: 0x00e6, 0x1f26: 0x00e6, 0x1f27: 0x00e6, 0x1f28: 0x00e6, 0x1f29: 0x00e6, - 0x1f2a: 0x00e6, 0x1f2b: 0x00e6, 0x1f2c: 0x00e6, 0x1f2d: 0x00e6, 0x1f2e: 0x00e6, 0x1f2f: 0x00e6, - 0x1f30: 0x00e6, 0x1f31: 0x00e6, - // Block 0x7d, offset 0x1f40 - 0x1f6b: 0x00dc, 0x1f6c: 0x00dc, 0x1f6d: 0x00dc, - // Block 0x7e, offset 0x1f80 - 0x1f93: 0x0009, - // Block 0x7f, offset 0x1fc0 - 0x1ff3: 0x0007, - // Block 0x80, offset 0x2000 - 0x2000: 0x0009, - // Block 0x81, offset 0x2040 - 0x2070: 0x00e6, 0x2072: 0x00e6, 0x2073: 0x00e6, 0x2074: 0x00dc, - 0x2077: 0x00e6, 0x2078: 0x00e6, - 0x207e: 0x00e6, 0x207f: 0x00e6, - // Block 0x82, offset 0x2080 - 0x2081: 0x00e6, - // Block 0x83, offset 0x20c0 - 0x20ed: 0x0009, - // Block 0x84, offset 0x2100 - 0x2100: 0x1100, 0x2101: 0x1100, 0x2102: 0x1100, 0x2103: 0x1100, 0x2104: 0x1100, 0x2105: 0x1100, - 0x2106: 0x1100, 0x2107: 0x1100, 0x2108: 0x1100, 0x2109: 0x1100, 0x210a: 0x1100, 0x210b: 0x1100, - 0x210c: 0x1100, 0x210d: 0x1100, 0x210e: 0x1100, 0x210f: 0x1100, 0x2110: 0x1100, 0x2111: 0x1100, - 0x2112: 0x1100, 0x2113: 0x1100, 0x2114: 0x1100, 0x2115: 0x1100, 0x2116: 0x1100, 0x2117: 0x1100, - 0x2118: 0x1100, 0x2119: 0x1100, 0x211a: 0x1100, 0x211b: 0x1100, 0x211c: 0x1100, 0x211d: 0x1100, - 0x211e: 0x1100, 0x211f: 0x1100, 0x2120: 0x1100, 0x2121: 0x1100, 0x2122: 0x1100, 0x2123: 0x1100, - 0x2124: 0x1100, 0x2125: 0x1100, 0x2126: 0x1100, 0x2127: 0x1100, 0x2128: 0x1100, 0x2129: 0x1100, - 0x212a: 0x1100, 0x212b: 0x1100, 0x212c: 0x1100, 0x212d: 0x1100, 0x212e: 0x1100, 0x212f: 0x1100, - 0x2130: 0x1100, 0x2131: 0x1100, 0x2132: 0x1100, 0x2133: 0x1100, 0x2134: 0x1100, 0x2135: 0x1100, - 0x2136: 0x1100, 0x2137: 0x1100, 0x2138: 0x1100, 0x2139: 0x1100, 0x213a: 0x1100, 0x213b: 0x1100, - 0x213c: 0x1100, 0x213d: 0x1100, 0x213e: 0x1100, 0x213f: 0x1100, - // Block 0x85, offset 0x2140 - 0x2140: 0x1100, 0x2141: 0x1100, 0x2142: 0x1100, 0x2143: 0x1100, 0x2144: 0x1100, 0x2145: 0x1100, - 0x2146: 0x1100, 0x2147: 0x1100, 0x2148: 0x1100, 0x2149: 0x1100, 0x214a: 0x1100, 0x214b: 0x1100, - 0x214c: 0x1100, 0x214d: 0x1100, 0x214e: 0x1100, 0x214f: 0x1100, 0x2150: 0x1100, 0x2151: 0x1100, - 0x2152: 0x1100, 0x2153: 0x1100, 0x2154: 0x1100, 0x2155: 0x1100, 0x2156: 0x1100, 0x2157: 0x1100, - 0x2158: 0x1100, 0x2159: 0x1100, 0x215a: 0x1100, 0x215b: 0x1100, 0x215c: 0x1100, 0x215d: 0x1100, - 0x215e: 0x1100, 0x215f: 0x1100, 0x2160: 0x1100, 0x2161: 0x1100, 0x2162: 0x1100, 0x2163: 0x1100, - // Block 0x86, offset 0x2180 - 0x2180: 0x3300, 0x2181: 0x3300, 0x2182: 0x3300, 0x2183: 0x3300, 0x2184: 0x3300, 0x2185: 0x3300, - 0x2186: 0x3300, 0x2187: 0x3300, 0x2188: 0x3300, 0x2189: 0x3300, 0x218a: 0x3300, 0x218b: 0x3300, - 0x218c: 0x3300, 0x218d: 0x3300, 0x218e: 0x3300, 0x218f: 0x3300, 0x2190: 0x3300, 0x2191: 0x3300, - 0x2192: 0x3300, 0x2193: 0x3300, 0x2194: 0x3300, 0x2195: 0x3300, 0x2196: 0x3300, 0x2197: 0x3300, - 0x2198: 0x3300, 0x2199: 0x3300, 0x219a: 0x3300, 0x219b: 0x3300, 0x219c: 0x3300, 0x219d: 0x3300, - 0x219e: 0x3300, 0x219f: 0x3300, 0x21a0: 0x3300, 0x21a1: 0x3300, 0x21a2: 0x3300, 0x21a3: 0x3300, - 0x21a4: 0x3300, 0x21a5: 0x3300, 0x21a6: 0x3300, 0x21a7: 0x3300, 0x21a8: 0x3300, 0x21a9: 0x3300, - 0x21aa: 0x3300, 0x21ab: 0x3300, 0x21ac: 0x3300, 0x21ad: 0x3300, 0x21ae: 0x3300, 0x21af: 0x3300, - 0x21b0: 0x3300, 0x21b1: 0x3300, 0x21b2: 0x3300, 0x21b3: 0x3300, 0x21b4: 0x3300, 0x21b5: 0x3300, - 0x21b6: 0x3300, 0x21b7: 0x3300, 0x21b8: 0x3300, 0x21b9: 0x3300, 0x21ba: 0x3300, 0x21bb: 0x3300, - 0x21bc: 0x3300, 0x21bd: 0x3300, 0x21be: 0x3300, 0x21bf: 0x3300, - // Block 0x87, offset 0x21c0 - 0x21c0: 0x3300, 0x21c1: 0x3300, 0x21c2: 0x3300, 0x21c3: 0x3300, 0x21c4: 0x3300, 0x21c5: 0x3300, - 0x21c6: 0x3300, 0x21c7: 0x3300, 0x21c8: 0x3300, 0x21c9: 0x3300, 0x21ca: 0x3300, 0x21cb: 0x3300, - 0x21cc: 0x3300, 0x21cd: 0x3300, 0x21d0: 0x3300, - 0x21d2: 0x3300, 0x21d5: 0x3300, 0x21d6: 0x3300, 0x21d7: 0x3300, - 0x21d8: 0x3300, 0x21d9: 0x3300, 0x21da: 0x3300, 0x21db: 0x3300, 0x21dc: 0x3300, 0x21dd: 0x3300, - 0x21de: 0x3300, 0x21e0: 0x3300, 0x21e2: 0x3300, - 0x21e5: 0x3300, 0x21e6: 0x3300, - 0x21ea: 0x3300, 0x21eb: 0x3300, 0x21ec: 0x3300, 0x21ed: 0x3300, - 0x21f0: 0x3300, 0x21f1: 0x3300, 0x21f2: 0x3300, 0x21f3: 0x3300, 0x21f4: 0x3300, 0x21f5: 0x3300, - 0x21f6: 0x3300, 0x21f7: 0x3300, 0x21f8: 0x3300, 0x21f9: 0x3300, 0x21fa: 0x3300, 0x21fb: 0x3300, - 0x21fc: 0x3300, 0x21fd: 0x3300, 0x21fe: 0x3300, 0x21ff: 0x3300, - // Block 0x88, offset 0x2200 - 0x2200: 0x3300, 0x2201: 0x3300, 0x2202: 0x3300, 0x2203: 0x3300, 0x2204: 0x3300, 0x2205: 0x3300, - 0x2206: 0x3300, 0x2207: 0x3300, 0x2208: 0x3300, 0x2209: 0x3300, 0x220a: 0x3300, 0x220b: 0x3300, - 0x220c: 0x3300, 0x220d: 0x3300, 0x220e: 0x3300, 0x220f: 0x3300, 0x2210: 0x3300, 0x2211: 0x3300, - 0x2212: 0x3300, 0x2213: 0x3300, 0x2214: 0x3300, 0x2215: 0x3300, 0x2216: 0x3300, 0x2217: 0x3300, - 0x2218: 0x3300, 0x2219: 0x3300, 0x221a: 0x3300, 0x221b: 0x3300, 0x221c: 0x3300, 0x221d: 0x3300, - 0x221e: 0x3300, 0x221f: 0x3300, 0x2220: 0x3300, 0x2221: 0x3300, 0x2222: 0x3300, 0x2223: 0x3300, - 0x2224: 0x3300, 0x2225: 0x3300, 0x2226: 0x3300, 0x2227: 0x3300, 0x2228: 0x3300, 0x2229: 0x3300, - 0x222a: 0x3300, 0x222b: 0x3300, 0x222c: 0x3300, 0x222d: 0x3300, - 0x2230: 0x3300, 0x2231: 0x3300, 0x2232: 0x3300, 0x2233: 0x3300, 0x2234: 0x3300, 0x2235: 0x3300, - 0x2236: 0x3300, 0x2237: 0x3300, 0x2238: 0x3300, 0x2239: 0x3300, 0x223a: 0x3300, 0x223b: 0x3300, - 0x223c: 0x3300, 0x223d: 0x3300, 0x223e: 0x3300, 0x223f: 0x3300, - // Block 0x89, offset 0x2240 - 0x2240: 0x3300, 0x2241: 0x3300, 0x2242: 0x3300, 0x2243: 0x3300, 0x2244: 0x3300, 0x2245: 0x3300, - 0x2246: 0x3300, 0x2247: 0x3300, 0x2248: 0x3300, 0x2249: 0x3300, 0x224a: 0x3300, 0x224b: 0x3300, - 0x224c: 0x3300, 0x224d: 0x3300, 0x224e: 0x3300, 0x224f: 0x3300, 0x2250: 0x3300, 0x2251: 0x3300, - 0x2252: 0x3300, 0x2253: 0x3300, 0x2254: 0x3300, 0x2255: 0x3300, 0x2256: 0x3300, 0x2257: 0x3300, - 0x2258: 0x3300, 0x2259: 0x3300, - // Block 0x8a, offset 0x2280 - 0x2280: 0x3000, 0x2281: 0x3000, 0x2282: 0x3000, 0x2283: 0x3000, 0x2284: 0x3000, 0x2285: 0x3000, - 0x2286: 0x3000, - 0x2293: 0x3000, 0x2294: 0x3000, 0x2295: 0x3000, 0x2296: 0x3000, 0x2297: 0x3000, - 0x229d: 0x3300, - 0x229e: 0x001a, 0x229f: 0x3300, 0x22a0: 0x3000, 0x22a1: 0x3000, 0x22a2: 0x3000, 0x22a3: 0x3000, - 0x22a4: 0x3000, 0x22a5: 0x3000, 0x22a6: 0x3000, 0x22a7: 0x3000, 0x22a8: 0x3000, 0x22a9: 0x3000, - 0x22aa: 0x3300, 0x22ab: 0x3300, 0x22ac: 0x3300, 0x22ad: 0x3300, 0x22ae: 0x3300, 0x22af: 0x3300, - 0x22b0: 0x3300, 0x22b1: 0x3300, 0x22b2: 0x3300, 0x22b3: 0x3300, 0x22b4: 0x3300, 0x22b5: 0x3300, - 0x22b6: 0x3300, 0x22b8: 0x3300, 0x22b9: 0x3300, 0x22ba: 0x3300, 0x22bb: 0x3300, - 0x22bc: 0x3300, 0x22be: 0x3300, - // Block 0x8b, offset 0x22c0 - 0x22c0: 0x3300, 0x22c1: 0x3300, 0x22c3: 0x3300, 0x22c4: 0x3300, - 0x22c6: 0x3300, 0x22c7: 0x3300, 0x22c8: 0x3300, 0x22c9: 0x3300, 0x22ca: 0x3300, 0x22cb: 0x3300, - 0x22cc: 0x3300, 0x22cd: 0x3300, 0x22ce: 0x3300, 0x22cf: 0x3000, 0x22d0: 0x3000, 0x22d1: 0x3000, - 0x22d2: 0x3000, 0x22d3: 0x3000, 0x22d4: 0x3000, 0x22d5: 0x3000, 0x22d6: 0x3000, 0x22d7: 0x3000, - 0x22d8: 0x3000, 0x22d9: 0x3000, 0x22da: 0x3000, 0x22db: 0x3000, 0x22dc: 0x3000, 0x22dd: 0x3000, - 0x22de: 0x3000, 0x22df: 0x3000, 0x22e0: 0x3000, 0x22e1: 0x3000, 0x22e2: 0x3000, 0x22e3: 0x3000, - 0x22e4: 0x3000, 0x22e5: 0x3000, 0x22e6: 0x3000, 0x22e7: 0x3000, 0x22e8: 0x3000, 0x22e9: 0x3000, - 0x22ea: 0x3000, 0x22eb: 0x3000, 0x22ec: 0x3000, 0x22ed: 0x3000, 0x22ee: 0x3000, 0x22ef: 0x3000, - 0x22f0: 0x3000, 0x22f1: 0x3000, 0x22f2: 0x3000, 0x22f3: 0x3000, 0x22f4: 0x3000, 0x22f5: 0x3000, - 0x22f6: 0x3000, 0x22f7: 0x3000, 0x22f8: 0x3000, 0x22f9: 0x3000, 0x22fa: 0x3000, 0x22fb: 0x3000, - 0x22fc: 0x3000, 0x22fd: 0x3000, 0x22fe: 0x3000, 0x22ff: 0x3000, - // Block 0x8c, offset 0x2300 - 0x2300: 0x3000, 0x2301: 0x3000, 0x2302: 0x3000, 0x2303: 0x3000, 0x2304: 0x3000, 0x2305: 0x3000, - 0x2306: 0x3000, 0x2307: 0x3000, 0x2308: 0x3000, 0x2309: 0x3000, 0x230a: 0x3000, 0x230b: 0x3000, - 0x230c: 0x3000, 0x230d: 0x3000, 0x230e: 0x3000, 0x230f: 0x3000, 0x2310: 0x3000, 0x2311: 0x3000, - 0x2312: 0x3000, 0x2313: 0x3000, 0x2314: 0x3000, 0x2315: 0x3000, 0x2316: 0x3000, 0x2317: 0x3000, - 0x2318: 0x3000, 0x2319: 0x3000, 0x231a: 0x3000, 0x231b: 0x3000, 0x231c: 0x3000, 0x231d: 0x3000, - 0x231e: 0x3000, 0x231f: 0x3000, 0x2320: 0x3000, 0x2321: 0x3000, 0x2322: 0x3000, 0x2323: 0x3000, - 0x2324: 0x3000, 0x2325: 0x3000, 0x2326: 0x3000, 0x2327: 0x3000, 0x2328: 0x3000, 0x2329: 0x3000, - 0x232a: 0x3000, 0x232b: 0x3000, 0x232c: 0x3000, 0x232d: 0x3000, 0x232e: 0x3000, 0x232f: 0x3000, - 0x2330: 0x3000, 0x2331: 0x3000, - // Block 0x8d, offset 0x2340 - 0x2353: 0x3000, 0x2354: 0x3000, 0x2355: 0x3000, 0x2356: 0x3000, 0x2357: 0x3000, - 0x2358: 0x3000, 0x2359: 0x3000, 0x235a: 0x3000, 0x235b: 0x3000, 0x235c: 0x3000, 0x235d: 0x3000, - 0x235e: 0x3000, 0x235f: 0x3000, 0x2360: 0x3000, 0x2361: 0x3000, 0x2362: 0x3000, 0x2363: 0x3000, - 0x2364: 0x3000, 0x2365: 0x3000, 0x2366: 0x3000, 0x2367: 0x3000, 0x2368: 0x3000, 0x2369: 0x3000, - 0x236a: 0x3000, 0x236b: 0x3000, 0x236c: 0x3000, 0x236d: 0x3000, 0x236e: 0x3000, 0x236f: 0x3000, - 0x2370: 0x3000, 0x2371: 0x3000, 0x2372: 0x3000, 0x2373: 0x3000, 0x2374: 0x3000, 0x2375: 0x3000, - 0x2376: 0x3000, 0x2377: 0x3000, 0x2378: 0x3000, 0x2379: 0x3000, 0x237a: 0x3000, 0x237b: 0x3000, - 0x237c: 0x3000, 0x237d: 0x3000, 0x237e: 0x3000, 0x237f: 0x3000, - // Block 0x8e, offset 0x2380 - 0x2380: 0x3000, 0x2381: 0x3000, 0x2382: 0x3000, 0x2383: 0x3000, 0x2384: 0x3000, 0x2385: 0x3000, - 0x2386: 0x3000, 0x2387: 0x3000, 0x2388: 0x3000, 0x2389: 0x3000, 0x238a: 0x3000, 0x238b: 0x3000, - 0x238c: 0x3000, 0x238d: 0x3000, 0x238e: 0x3000, 0x238f: 0x3000, 0x2390: 0x3000, 0x2391: 0x3000, - 0x2392: 0x3000, 0x2393: 0x3000, 0x2394: 0x3000, 0x2395: 0x3000, 0x2396: 0x3000, 0x2397: 0x3000, - 0x2398: 0x3000, 0x2399: 0x3000, 0x239a: 0x3000, 0x239b: 0x3000, 0x239c: 0x3000, 0x239d: 0x3000, - 0x239e: 0x3000, 0x239f: 0x3000, 0x23a0: 0x3000, 0x23a1: 0x3000, 0x23a2: 0x3000, 0x23a3: 0x3000, - 0x23a4: 0x3000, 0x23a5: 0x3000, 0x23a6: 0x3000, 0x23a7: 0x3000, 0x23a8: 0x3000, 0x23a9: 0x3000, - 0x23aa: 0x3000, 0x23ab: 0x3000, 0x23ac: 0x3000, 0x23ad: 0x3000, 0x23ae: 0x3000, 0x23af: 0x3000, - 0x23b0: 0x3000, 0x23b1: 0x3000, 0x23b2: 0x3000, 0x23b3: 0x3000, 0x23b4: 0x3000, 0x23b5: 0x3000, - 0x23b6: 0x3000, 0x23b7: 0x3000, 0x23b8: 0x3000, 0x23b9: 0x3000, 0x23ba: 0x3000, 0x23bb: 0x3000, - 0x23bc: 0x3000, 0x23bd: 0x3000, - // Block 0x8f, offset 0x23c0 - 0x23d0: 0x3000, 0x23d1: 0x3000, - 0x23d2: 0x3000, 0x23d3: 0x3000, 0x23d4: 0x3000, 0x23d5: 0x3000, 0x23d6: 0x3000, 0x23d7: 0x3000, - 0x23d8: 0x3000, 0x23d9: 0x3000, 0x23da: 0x3000, 0x23db: 0x3000, 0x23dc: 0x3000, 0x23dd: 0x3000, - 0x23de: 0x3000, 0x23df: 0x3000, 0x23e0: 0x3000, 0x23e1: 0x3000, 0x23e2: 0x3000, 0x23e3: 0x3000, - 0x23e4: 0x3000, 0x23e5: 0x3000, 0x23e6: 0x3000, 0x23e7: 0x3000, 0x23e8: 0x3000, 0x23e9: 0x3000, - 0x23ea: 0x3000, 0x23eb: 0x3000, 0x23ec: 0x3000, 0x23ed: 0x3000, 0x23ee: 0x3000, 0x23ef: 0x3000, - 0x23f0: 0x3000, 0x23f1: 0x3000, 0x23f2: 0x3000, 0x23f3: 0x3000, 0x23f4: 0x3000, 0x23f5: 0x3000, - 0x23f6: 0x3000, 0x23f7: 0x3000, 0x23f8: 0x3000, 0x23f9: 0x3000, 0x23fa: 0x3000, 0x23fb: 0x3000, - 0x23fc: 0x3000, 0x23fd: 0x3000, 0x23fe: 0x3000, 0x23ff: 0x3000, - // Block 0x90, offset 0x2400 - 0x2400: 0x3000, 0x2401: 0x3000, 0x2402: 0x3000, 0x2403: 0x3000, 0x2404: 0x3000, 0x2405: 0x3000, - 0x2406: 0x3000, 0x2407: 0x3000, 0x2408: 0x3000, 0x2409: 0x3000, 0x240a: 0x3000, 0x240b: 0x3000, - 0x240c: 0x3000, 0x240d: 0x3000, 0x240e: 0x3000, 0x240f: 0x3000, - 0x2412: 0x3000, 0x2413: 0x3000, 0x2414: 0x3000, 0x2415: 0x3000, 0x2416: 0x3000, 0x2417: 0x3000, - 0x2418: 0x3000, 0x2419: 0x3000, 0x241a: 0x3000, 0x241b: 0x3000, 0x241c: 0x3000, 0x241d: 0x3000, - 0x241e: 0x3000, 0x241f: 0x3000, 0x2420: 0x3000, 0x2421: 0x3000, 0x2422: 0x3000, 0x2423: 0x3000, - 0x2424: 0x3000, 0x2425: 0x3000, 0x2426: 0x3000, 0x2427: 0x3000, 0x2428: 0x3000, 0x2429: 0x3000, - 0x242a: 0x3000, 0x242b: 0x3000, 0x242c: 0x3000, 0x242d: 0x3000, 0x242e: 0x3000, 0x242f: 0x3000, - 0x2430: 0x3000, 0x2431: 0x3000, 0x2432: 0x3000, 0x2433: 0x3000, 0x2434: 0x3000, 0x2435: 0x3000, - 0x2436: 0x3000, 0x2437: 0x3000, 0x2438: 0x3000, 0x2439: 0x3000, 0x243a: 0x3000, 0x243b: 0x3000, - 0x243c: 0x3000, 0x243d: 0x3000, 0x243e: 0x3000, 0x243f: 0x3000, - // Block 0x91, offset 0x2440 - 0x2440: 0x3000, 0x2441: 0x3000, 0x2442: 0x3000, 0x2443: 0x3000, 0x2444: 0x3000, 0x2445: 0x3000, - 0x2446: 0x3000, 0x2447: 0x3000, - 0x2470: 0x3000, 0x2471: 0x3000, 0x2472: 0x3000, 0x2473: 0x3000, 0x2474: 0x3000, 0x2475: 0x3000, - 0x2476: 0x3000, 0x2477: 0x3000, 0x2478: 0x3000, 0x2479: 0x3000, 0x247a: 0x3000, 0x247b: 0x3000, - 0x247c: 0x3000, - // Block 0x92, offset 0x2480 - 0x2490: 0x3000, 0x2491: 0x3000, - 0x2492: 0x3000, 0x2493: 0x3000, 0x2494: 0x3000, 0x2495: 0x3000, 0x2496: 0x3000, 0x2497: 0x3000, - 0x2498: 0x3000, 0x2499: 0x3000, - 0x24a0: 0x00e6, 0x24a1: 0x00e6, 0x24a2: 0x00e6, 0x24a3: 0x00e6, - 0x24a4: 0x00e6, 0x24a5: 0x00e6, 0x24a6: 0x00e6, - 0x24b0: 0x3000, 0x24b1: 0x3000, 0x24b2: 0x3000, 0x24b3: 0x3000, 0x24b4: 0x3000, 0x24b5: 0x3000, - 0x24b6: 0x3000, 0x24b7: 0x3000, 0x24b8: 0x3000, 0x24b9: 0x3000, 0x24ba: 0x3000, 0x24bb: 0x3000, - 0x24bc: 0x3000, 0x24bd: 0x3000, 0x24be: 0x3000, 0x24bf: 0x3000, - // Block 0x93, offset 0x24c0 - 0x24c0: 0x3000, 0x24c1: 0x3000, 0x24c2: 0x3000, 0x24c3: 0x3000, 0x24c4: 0x3000, - 0x24c7: 0x3000, 0x24c8: 0x3000, 0x24c9: 0x3000, 0x24ca: 0x3000, 0x24cb: 0x3000, - 0x24cc: 0x3000, 0x24cd: 0x3000, 0x24ce: 0x3000, 0x24cf: 0x3000, 0x24d0: 0x3000, 0x24d1: 0x3000, - 0x24d2: 0x3000, 0x24d4: 0x3000, 0x24d5: 0x3000, 0x24d6: 0x3000, 0x24d7: 0x3000, - 0x24d8: 0x3000, 0x24d9: 0x3000, 0x24da: 0x3000, 0x24db: 0x3000, 0x24dc: 0x3000, 0x24dd: 0x3000, - 0x24de: 0x3000, 0x24df: 0x3000, 0x24e0: 0x3000, 0x24e1: 0x3000, 0x24e2: 0x3000, 0x24e3: 0x3000, - 0x24e4: 0x3000, 0x24e5: 0x3000, 0x24e6: 0x3000, 0x24e8: 0x3000, 0x24e9: 0x3000, - 0x24ea: 0x3000, 0x24eb: 0x3000, - 0x24f0: 0x3000, 0x24f1: 0x3000, 0x24f2: 0x3000, 0x24f4: 0x3000, - 0x24f6: 0x3000, 0x24f7: 0x3000, 0x24f8: 0x3000, 0x24f9: 0x3000, 0x24fa: 0x3000, 0x24fb: 0x3000, - 0x24fc: 0x3000, 0x24fd: 0x3000, 0x24fe: 0x3000, 0x24ff: 0x3000, - // Block 0x94, offset 0x2500 - 0x2500: 0x3000, 0x2501: 0x3000, 0x2502: 0x3000, 0x2503: 0x3000, 0x2504: 0x3000, 0x2505: 0x3000, - 0x2506: 0x3000, 0x2507: 0x3000, 0x2508: 0x3000, 0x2509: 0x3000, 0x250a: 0x3000, 0x250b: 0x3000, - 0x250c: 0x3000, 0x250d: 0x3000, 0x250e: 0x3000, 0x250f: 0x3000, 0x2510: 0x3000, 0x2511: 0x3000, - 0x2512: 0x3000, 0x2513: 0x3000, 0x2514: 0x3000, 0x2515: 0x3000, 0x2516: 0x3000, 0x2517: 0x3000, - 0x2518: 0x3000, 0x2519: 0x3000, 0x251a: 0x3000, 0x251b: 0x3000, 0x251c: 0x3000, 0x251d: 0x3000, - 0x251e: 0x3000, 0x251f: 0x3000, 0x2520: 0x3000, 0x2521: 0x3000, 0x2522: 0x3000, 0x2523: 0x3000, - 0x2524: 0x3000, 0x2525: 0x3000, 0x2526: 0x3000, 0x2527: 0x3000, 0x2528: 0x3000, 0x2529: 0x3000, - 0x252a: 0x3000, 0x252b: 0x3000, 0x252c: 0x3000, 0x252d: 0x3000, 0x252e: 0x3000, 0x252f: 0x3000, - 0x2530: 0x3000, 0x2531: 0x3000, 0x2532: 0x3000, 0x2533: 0x3000, 0x2534: 0x3000, 0x2535: 0x3000, - 0x2536: 0x3000, 0x2537: 0x3000, 0x2538: 0x3000, 0x2539: 0x3000, 0x253a: 0x3000, 0x253b: 0x3000, - 0x253c: 0x3000, - // Block 0x95, offset 0x2540 - 0x2541: 0x3000, 0x2542: 0x3000, 0x2543: 0x3000, 0x2544: 0x3000, 0x2545: 0x3000, - 0x2546: 0x3000, 0x2547: 0x3000, 0x2548: 0x3000, 0x2549: 0x3000, 0x254a: 0x3000, 0x254b: 0x3000, - 0x254c: 0x3000, 0x254d: 0x3000, 0x254e: 0x3000, 0x254f: 0x3000, 0x2550: 0x3000, 0x2551: 0x3000, - 0x2552: 0x3000, 0x2553: 0x3000, 0x2554: 0x3000, 0x2555: 0x3000, 0x2556: 0x3000, 0x2557: 0x3000, - 0x2558: 0x3000, 0x2559: 0x3000, 0x255a: 0x3000, 0x255b: 0x3000, 0x255c: 0x3000, 0x255d: 0x3000, - 0x255e: 0x3000, 0x255f: 0x3000, 0x2560: 0x3000, 0x2561: 0x3000, 0x2562: 0x3000, 0x2563: 0x3000, - 0x2564: 0x3000, 0x2565: 0x3000, 0x2566: 0x3000, 0x2567: 0x3000, 0x2568: 0x3000, 0x2569: 0x3000, - 0x256a: 0x3000, 0x256b: 0x3000, 0x256c: 0x3000, 0x256d: 0x3000, 0x256e: 0x3000, 0x256f: 0x3000, - 0x2570: 0x3000, 0x2571: 0x3000, 0x2572: 0x3000, 0x2573: 0x3000, 0x2574: 0x3000, 0x2575: 0x3000, - 0x2576: 0x3000, 0x2577: 0x3000, 0x2578: 0x3000, 0x2579: 0x3000, 0x257a: 0x3000, 0x257b: 0x3000, - 0x257c: 0x3000, 0x257d: 0x3000, 0x257e: 0x3000, 0x257f: 0x3000, - // Block 0x96, offset 0x2580 - 0x2582: 0x3000, 0x2583: 0x3000, 0x2584: 0x3000, 0x2585: 0x3000, - 0x2586: 0x3000, 0x2587: 0x3000, 0x258a: 0x3000, 0x258b: 0x3000, - 0x258c: 0x3000, 0x258d: 0x3000, 0x258e: 0x3000, 0x258f: 0x3000, - 0x2592: 0x3000, 0x2593: 0x3000, 0x2594: 0x3000, 0x2595: 0x3000, 0x2596: 0x3000, 0x2597: 0x3000, - 0x259a: 0x3000, 0x259b: 0x3000, 0x259c: 0x3000, - 0x25a0: 0x3000, 0x25a1: 0x3000, 0x25a2: 0x3000, 0x25a3: 0x3000, - 0x25a4: 0x3000, 0x25a5: 0x3000, 0x25a6: 0x3000, 0x25a8: 0x3000, 0x25a9: 0x3000, - 0x25aa: 0x3000, 0x25ab: 0x3000, 0x25ac: 0x3000, 0x25ad: 0x3000, 0x25ae: 0x3000, - // Block 0x97, offset 0x25c0 - 0x25fd: 0x00dc, - // Block 0x98, offset 0x2600 - 0x260d: 0x00dc, 0x260f: 0x00e6, - 0x2638: 0x00e6, 0x2639: 0x0001, 0x263a: 0x00dc, - 0x263f: 0x0009, - // Block 0x99, offset 0x2640 - 0x2659: 0x8800, 0x265a: 0x1100, 0x265b: 0x8800, 0x265c: 0x1100, - 0x2665: 0x8800, - 0x266b: 0x1100, - 0x2679: 0x0009, 0x267a: 0x6607, - // Block 0x9a, offset 0x2680 - 0x269e: 0x3300, 0x269f: 0x3300, 0x26a0: 0x3300, 0x26a1: 0x3300, 0x26a2: 0x3300, 0x26a3: 0x3300, - 0x26a4: 0x3300, 0x26a5: 0x00d8, 0x26a6: 0x00d8, 0x26a7: 0x0001, 0x26a8: 0x0001, 0x26a9: 0x0001, - 0x26ad: 0x00e2, 0x26ae: 0x00d8, 0x26af: 0x00d8, - 0x26b0: 0x00d8, 0x26b1: 0x00d8, 0x26b2: 0x00d8, - 0x26bb: 0x00dc, - 0x26bc: 0x00dc, 0x26bd: 0x00dc, 0x26be: 0x00dc, 0x26bf: 0x00dc, - // Block 0x9b, offset 0x26c0 - 0x26c0: 0x00dc, 0x26c1: 0x00dc, 0x26c2: 0x00dc, 0x26c5: 0x00e6, - 0x26c6: 0x00e6, 0x26c7: 0x00e6, 0x26c8: 0x00e6, 0x26c9: 0x00e6, 0x26ca: 0x00dc, 0x26cb: 0x00dc, - 0x26ea: 0x00e6, 0x26eb: 0x00e6, 0x26ec: 0x00e6, 0x26ed: 0x00e6, - 0x26fb: 0x3300, - 0x26fc: 0x3300, 0x26fd: 0x3300, 0x26fe: 0x3300, 0x26ff: 0x3300, - // Block 0x9c, offset 0x2700 - 0x2700: 0x3300, - // Block 0x9d, offset 0x2740 - 0x2742: 0x00e6, 0x2743: 0x00e6, 0x2744: 0x00e6, - // Block 0x9e, offset 0x2780 - 0x2780: 0x3000, 0x2781: 0x3000, 0x2782: 0x3000, 0x2783: 0x3000, 0x2784: 0x3000, 0x2785: 0x3000, - 0x2786: 0x3000, 0x2787: 0x3000, 0x2788: 0x3000, 0x2789: 0x3000, 0x278a: 0x3000, 0x278b: 0x3000, - 0x278c: 0x3000, 0x278d: 0x3000, 0x278e: 0x3000, 0x278f: 0x3000, 0x2790: 0x3000, 0x2791: 0x3000, - 0x2792: 0x3000, 0x2793: 0x3000, 0x2794: 0x3000, 0x2796: 0x3000, 0x2797: 0x3000, - 0x2798: 0x3000, 0x2799: 0x3000, 0x279a: 0x3000, 0x279b: 0x3000, 0x279c: 0x3000, 0x279d: 0x3000, - 0x279e: 0x3000, 0x279f: 0x3000, 0x27a0: 0x3000, 0x27a1: 0x3000, 0x27a2: 0x3000, 0x27a3: 0x3000, - 0x27a4: 0x3000, 0x27a5: 0x3000, 0x27a6: 0x3000, 0x27a7: 0x3000, 0x27a8: 0x3000, 0x27a9: 0x3000, - 0x27aa: 0x3000, 0x27ab: 0x3000, 0x27ac: 0x3000, 0x27ad: 0x3000, 0x27ae: 0x3000, 0x27af: 0x3000, - 0x27b0: 0x3000, 0x27b1: 0x3000, 0x27b2: 0x3000, 0x27b3: 0x3000, 0x27b4: 0x3000, 0x27b5: 0x3000, - 0x27b6: 0x3000, 0x27b7: 0x3000, 0x27b8: 0x3000, 0x27b9: 0x3000, 0x27ba: 0x3000, 0x27bb: 0x3000, - 0x27bc: 0x3000, 0x27bd: 0x3000, 0x27be: 0x3000, 0x27bf: 0x3000, - // Block 0x9f, offset 0x27c0 - 0x27c0: 0x3000, 0x27c1: 0x3000, 0x27c2: 0x3000, 0x27c3: 0x3000, 0x27c4: 0x3000, 0x27c5: 0x3000, - 0x27c6: 0x3000, 0x27c7: 0x3000, 0x27c8: 0x3000, 0x27c9: 0x3000, 0x27ca: 0x3000, 0x27cb: 0x3000, - 0x27cc: 0x3000, 0x27cd: 0x3000, 0x27ce: 0x3000, 0x27cf: 0x3000, 0x27d0: 0x3000, 0x27d1: 0x3000, - 0x27d2: 0x3000, 0x27d3: 0x3000, 0x27d4: 0x3000, 0x27d5: 0x3000, 0x27d6: 0x3000, 0x27d7: 0x3000, - 0x27d8: 0x3000, 0x27d9: 0x3000, 0x27da: 0x3000, 0x27db: 0x3000, 0x27dc: 0x3000, - 0x27de: 0x3000, 0x27df: 0x3000, 0x27e2: 0x3000, - 0x27e5: 0x3000, 0x27e6: 0x3000, 0x27e9: 0x3000, - 0x27ea: 0x3000, 0x27eb: 0x3000, 0x27ec: 0x3000, 0x27ee: 0x3000, 0x27ef: 0x3000, - 0x27f0: 0x3000, 0x27f1: 0x3000, 0x27f2: 0x3000, 0x27f3: 0x3000, 0x27f4: 0x3000, 0x27f5: 0x3000, - 0x27f6: 0x3000, 0x27f7: 0x3000, 0x27f8: 0x3000, 0x27f9: 0x3000, 0x27fb: 0x3000, - 0x27fd: 0x3000, 0x27fe: 0x3000, 0x27ff: 0x3000, - // Block 0xa0, offset 0x2800 - 0x2800: 0x3000, 0x2801: 0x3000, 0x2802: 0x3000, 0x2803: 0x3000, 0x2805: 0x3000, - 0x2806: 0x3000, 0x2807: 0x3000, 0x2808: 0x3000, 0x2809: 0x3000, 0x280a: 0x3000, 0x280b: 0x3000, - 0x280c: 0x3000, 0x280d: 0x3000, 0x280e: 0x3000, 0x280f: 0x3000, 0x2810: 0x3000, 0x2811: 0x3000, - 0x2812: 0x3000, 0x2813: 0x3000, 0x2814: 0x3000, 0x2815: 0x3000, 0x2816: 0x3000, 0x2817: 0x3000, - 0x2818: 0x3000, 0x2819: 0x3000, 0x281a: 0x3000, 0x281b: 0x3000, 0x281c: 0x3000, 0x281d: 0x3000, - 0x281e: 0x3000, 0x281f: 0x3000, 0x2820: 0x3000, 0x2821: 0x3000, 0x2822: 0x3000, 0x2823: 0x3000, - 0x2824: 0x3000, 0x2825: 0x3000, 0x2826: 0x3000, 0x2827: 0x3000, 0x2828: 0x3000, 0x2829: 0x3000, - 0x282a: 0x3000, 0x282b: 0x3000, 0x282c: 0x3000, 0x282d: 0x3000, 0x282e: 0x3000, 0x282f: 0x3000, - 0x2830: 0x3000, 0x2831: 0x3000, 0x2832: 0x3000, 0x2833: 0x3000, 0x2834: 0x3000, 0x2835: 0x3000, - 0x2836: 0x3000, 0x2837: 0x3000, 0x2838: 0x3000, 0x2839: 0x3000, 0x283a: 0x3000, 0x283b: 0x3000, - 0x283c: 0x3000, 0x283d: 0x3000, 0x283e: 0x3000, 0x283f: 0x3000, - // Block 0xa1, offset 0x2840 - 0x2840: 0x3000, 0x2841: 0x3000, 0x2842: 0x3000, 0x2843: 0x3000, 0x2844: 0x3000, 0x2845: 0x3000, - 0x2847: 0x3000, 0x2848: 0x3000, 0x2849: 0x3000, 0x284a: 0x3000, - 0x284d: 0x3000, 0x284e: 0x3000, 0x284f: 0x3000, 0x2850: 0x3000, 0x2851: 0x3000, - 0x2852: 0x3000, 0x2853: 0x3000, 0x2854: 0x3000, 0x2856: 0x3000, 0x2857: 0x3000, - 0x2858: 0x3000, 0x2859: 0x3000, 0x285a: 0x3000, 0x285b: 0x3000, 0x285c: 0x3000, - 0x285e: 0x3000, 0x285f: 0x3000, 0x2860: 0x3000, 0x2861: 0x3000, 0x2862: 0x3000, 0x2863: 0x3000, - 0x2864: 0x3000, 0x2865: 0x3000, 0x2866: 0x3000, 0x2867: 0x3000, 0x2868: 0x3000, 0x2869: 0x3000, - 0x286a: 0x3000, 0x286b: 0x3000, 0x286c: 0x3000, 0x286d: 0x3000, 0x286e: 0x3000, 0x286f: 0x3000, - 0x2870: 0x3000, 0x2871: 0x3000, 0x2872: 0x3000, 0x2873: 0x3000, 0x2874: 0x3000, 0x2875: 0x3000, - 0x2876: 0x3000, 0x2877: 0x3000, 0x2878: 0x3000, 0x2879: 0x3000, 0x287b: 0x3000, - 0x287c: 0x3000, 0x287d: 0x3000, 0x287e: 0x3000, - // Block 0xa2, offset 0x2880 - 0x2880: 0x3000, 0x2881: 0x3000, 0x2882: 0x3000, 0x2883: 0x3000, 0x2884: 0x3000, - 0x2886: 0x3000, 0x288a: 0x3000, 0x288b: 0x3000, - 0x288c: 0x3000, 0x288d: 0x3000, 0x288e: 0x3000, 0x288f: 0x3000, 0x2890: 0x3000, - 0x2892: 0x3000, 0x2893: 0x3000, 0x2894: 0x3000, 0x2895: 0x3000, 0x2896: 0x3000, 0x2897: 0x3000, - 0x2898: 0x3000, 0x2899: 0x3000, 0x289a: 0x3000, 0x289b: 0x3000, 0x289c: 0x3000, 0x289d: 0x3000, - 0x289e: 0x3000, 0x289f: 0x3000, 0x28a0: 0x3000, 0x28a1: 0x3000, 0x28a2: 0x3000, 0x28a3: 0x3000, - 0x28a4: 0x3000, 0x28a5: 0x3000, 0x28a6: 0x3000, 0x28a7: 0x3000, 0x28a8: 0x3000, 0x28a9: 0x3000, - 0x28aa: 0x3000, 0x28ab: 0x3000, 0x28ac: 0x3000, 0x28ad: 0x3000, 0x28ae: 0x3000, 0x28af: 0x3000, - 0x28b0: 0x3000, 0x28b1: 0x3000, 0x28b2: 0x3000, 0x28b3: 0x3000, 0x28b4: 0x3000, 0x28b5: 0x3000, - 0x28b6: 0x3000, 0x28b7: 0x3000, 0x28b8: 0x3000, 0x28b9: 0x3000, 0x28ba: 0x3000, 0x28bb: 0x3000, - 0x28bc: 0x3000, 0x28bd: 0x3000, 0x28be: 0x3000, 0x28bf: 0x3000, - // Block 0xa3, offset 0x28c0 - 0x28c0: 0x3000, 0x28c1: 0x3000, 0x28c2: 0x3000, 0x28c3: 0x3000, 0x28c4: 0x3000, 0x28c5: 0x3000, - 0x28c6: 0x3000, 0x28c7: 0x3000, 0x28c8: 0x3000, 0x28c9: 0x3000, 0x28ca: 0x3000, 0x28cb: 0x3000, - 0x28cc: 0x3000, 0x28cd: 0x3000, 0x28ce: 0x3000, 0x28cf: 0x3000, 0x28d0: 0x3000, 0x28d1: 0x3000, - 0x28d2: 0x3000, 0x28d3: 0x3000, 0x28d4: 0x3000, 0x28d5: 0x3000, 0x28d6: 0x3000, 0x28d7: 0x3000, - 0x28d8: 0x3000, 0x28d9: 0x3000, 0x28da: 0x3000, 0x28db: 0x3000, 0x28dc: 0x3000, 0x28dd: 0x3000, - 0x28de: 0x3000, 0x28df: 0x3000, 0x28e0: 0x3000, 0x28e1: 0x3000, 0x28e2: 0x3000, 0x28e3: 0x3000, - 0x28e4: 0x3000, 0x28e5: 0x3000, 0x28e8: 0x3000, 0x28e9: 0x3000, - 0x28ea: 0x3000, 0x28eb: 0x3000, 0x28ec: 0x3000, 0x28ed: 0x3000, 0x28ee: 0x3000, 0x28ef: 0x3000, - 0x28f0: 0x3000, 0x28f1: 0x3000, 0x28f2: 0x3000, 0x28f3: 0x3000, 0x28f4: 0x3000, 0x28f5: 0x3000, - 0x28f6: 0x3000, 0x28f7: 0x3000, 0x28f8: 0x3000, 0x28f9: 0x3000, 0x28fa: 0x3000, 0x28fb: 0x3000, - 0x28fc: 0x3000, 0x28fd: 0x3000, 0x28fe: 0x3000, 0x28ff: 0x3000, - // Block 0xa4, offset 0x2900 - 0x2900: 0x3000, 0x2901: 0x3000, 0x2902: 0x3000, 0x2903: 0x3000, 0x2904: 0x3000, 0x2905: 0x3000, - 0x2906: 0x3000, 0x2907: 0x3000, 0x2908: 0x3000, 0x2909: 0x3000, 0x290a: 0x3000, 0x290b: 0x3000, - 0x290e: 0x3000, 0x290f: 0x3000, 0x2910: 0x3000, 0x2911: 0x3000, - 0x2912: 0x3000, 0x2913: 0x3000, 0x2914: 0x3000, 0x2915: 0x3000, 0x2916: 0x3000, 0x2917: 0x3000, - 0x2918: 0x3000, 0x2919: 0x3000, 0x291a: 0x3000, 0x291b: 0x3000, 0x291c: 0x3000, 0x291d: 0x3000, - 0x291e: 0x3000, 0x291f: 0x3000, 0x2920: 0x3000, 0x2921: 0x3000, 0x2922: 0x3000, 0x2923: 0x3000, - 0x2924: 0x3000, 0x2925: 0x3000, 0x2926: 0x3000, 0x2927: 0x3000, 0x2928: 0x3000, 0x2929: 0x3000, - 0x292a: 0x3000, 0x292b: 0x3000, 0x292c: 0x3000, 0x292d: 0x3000, 0x292e: 0x3000, 0x292f: 0x3000, - 0x2930: 0x3000, 0x2931: 0x3000, 0x2932: 0x3000, 0x2933: 0x3000, 0x2934: 0x3000, 0x2935: 0x3000, - 0x2936: 0x3000, 0x2937: 0x3000, 0x2938: 0x3000, 0x2939: 0x3000, 0x293a: 0x3000, 0x293b: 0x3000, - 0x293c: 0x3000, 0x293d: 0x3000, 0x293e: 0x3000, 0x293f: 0x3000, - // Block 0xa5, offset 0x2940 - 0x2940: 0x3000, 0x2941: 0x3000, 0x2942: 0x3000, 0x2943: 0x3000, 0x2944: 0x3000, 0x2945: 0x3000, - 0x2946: 0x3000, 0x2947: 0x3000, 0x2948: 0x3000, 0x2949: 0x3000, 0x294a: 0x3000, - 0x2950: 0x3000, 0x2951: 0x3000, - 0x2952: 0x3000, 0x2953: 0x3000, 0x2954: 0x3000, 0x2955: 0x3000, 0x2956: 0x3000, 0x2957: 0x3000, - 0x2958: 0x3000, 0x2959: 0x3000, 0x295a: 0x3000, 0x295b: 0x3000, 0x295c: 0x3000, 0x295d: 0x3000, - 0x295e: 0x3000, 0x295f: 0x3000, 0x2960: 0x3000, 0x2961: 0x3000, 0x2962: 0x3000, 0x2963: 0x3000, - 0x2964: 0x3000, 0x2965: 0x3000, 0x2966: 0x3000, 0x2967: 0x3000, 0x2968: 0x3000, 0x2969: 0x3000, - 0x296a: 0x3000, 0x296b: 0x3000, 0x296c: 0x3000, 0x296d: 0x3000, 0x296e: 0x3000, - 0x2970: 0x3000, 0x2971: 0x3000, 0x2972: 0x3000, 0x2973: 0x3000, 0x2974: 0x3000, 0x2975: 0x3000, - 0x2976: 0x3000, 0x2977: 0x3000, 0x2978: 0x3000, 0x2979: 0x3000, 0x297a: 0x3000, 0x297b: 0x3000, - 0x297c: 0x3000, 0x297d: 0x3000, 0x297e: 0x3000, 0x297f: 0x3000, - // Block 0xa6, offset 0x2980 - 0x2980: 0x3000, 0x2981: 0x3000, 0x2982: 0x3000, 0x2983: 0x3000, 0x2984: 0x3000, 0x2985: 0x3000, - 0x2986: 0x3000, 0x2987: 0x3000, 0x2988: 0x3000, 0x2989: 0x3000, 0x298a: 0x3000, 0x298b: 0x3000, - 0x298c: 0x3000, 0x298d: 0x3000, 0x298e: 0x3000, 0x298f: 0x3000, - // Block 0xa7, offset 0x29c0 - 0x29d0: 0x3000, - // Block 0xa8, offset 0x2a00 - 0x2a00: 0x3000, 0x2a01: 0x3000, 0x2a02: 0x3000, - 0x2a10: 0x3000, 0x2a11: 0x3000, - 0x2a12: 0x3000, 0x2a13: 0x3000, 0x2a14: 0x3000, 0x2a15: 0x3000, 0x2a16: 0x3000, 0x2a17: 0x3000, - 0x2a18: 0x3000, 0x2a19: 0x3000, 0x2a1a: 0x3000, 0x2a1b: 0x3000, 0x2a1c: 0x3000, 0x2a1d: 0x3000, - 0x2a1e: 0x3000, 0x2a1f: 0x3000, 0x2a20: 0x3000, 0x2a21: 0x3000, 0x2a22: 0x3000, 0x2a23: 0x3000, - 0x2a24: 0x3000, 0x2a25: 0x3000, 0x2a26: 0x3000, 0x2a27: 0x3000, 0x2a28: 0x3000, 0x2a29: 0x3000, - 0x2a2a: 0x3000, 0x2a2b: 0x3000, 0x2a2c: 0x3000, 0x2a2d: 0x3000, 0x2a2e: 0x3000, 0x2a2f: 0x3000, - 0x2a30: 0x3000, 0x2a31: 0x3000, 0x2a32: 0x3000, 0x2a33: 0x3000, 0x2a34: 0x3000, 0x2a35: 0x3000, - 0x2a36: 0x3000, 0x2a37: 0x3000, 0x2a38: 0x3000, 0x2a39: 0x3000, 0x2a3a: 0x3000, - // Block 0xa9, offset 0x2a40 - 0x2a40: 0x3000, 0x2a41: 0x3000, 0x2a42: 0x3000, 0x2a43: 0x3000, 0x2a44: 0x3000, 0x2a45: 0x3000, - 0x2a46: 0x3000, 0x2a47: 0x3000, 0x2a48: 0x3000, - 0x2a50: 0x3000, 0x2a51: 0x3000, - // Block 0xaa, offset 0x2a80 - 0x2a80: 0x3300, 0x2a81: 0x3300, 0x2a82: 0x3300, 0x2a83: 0x3300, 0x2a84: 0x3300, 0x2a85: 0x3300, - 0x2a86: 0x3300, 0x2a87: 0x3300, 0x2a88: 0x3300, 0x2a89: 0x3300, 0x2a8a: 0x3300, 0x2a8b: 0x3300, - 0x2a8c: 0x3300, 0x2a8d: 0x3300, 0x2a8e: 0x3300, 0x2a8f: 0x3300, 0x2a90: 0x3300, 0x2a91: 0x3300, - 0x2a92: 0x3300, 0x2a93: 0x3300, 0x2a94: 0x3300, 0x2a95: 0x3300, 0x2a96: 0x3300, 0x2a97: 0x3300, - 0x2a98: 0x3300, 0x2a99: 0x3300, 0x2a9a: 0x3300, 0x2a9b: 0x3300, 0x2a9c: 0x3300, 0x2a9d: 0x3300, + 0x03c0: 0x1100, 0x03c1: 0x8800, 0x03c2: 0x1100, 0x03c4: 0x8800, 0x03c5: 0x1100, + 0x03c6: 0x8800, 0x03c7: 0x1100, 0x03c8: 0x8800, 0x03c9: 0x1100, + 0x03cf: 0x8800, 0x03d0: 0x1100, 0x03d1: 0x1100, + 0x03d2: 0x8800, 0x03d3: 0x1100, 0x03d4: 0x1100, 0x03d5: 0x8800, 0x03d6: 0x1100, 0x03d7: 0x1100, + 0x03d8: 0x8800, 0x03d9: 0x1100, 0x03da: 0x1100, 0x03db: 0x8800, 0x03dc: 0x1100, 0x03dd: 0x1100, + 0x03ef: 0x8800, + 0x03f0: 0x8800, 0x03f1: 0x8800, 0x03f2: 0x8800, 0x03f4: 0x1100, + 0x03f7: 0x1100, 0x03f8: 0x1100, 0x03f9: 0x1100, 0x03fa: 0x1100, + 0x03fd: 0x8800, 0x03fe: 0x1100, 0x03ff: 0x3000, +} + +// charInfoSparseOffset: 156 entries, 312 bytes +var charInfoSparseOffset = []uint16{0x0, 0x8, 0x13, 0x21, 0x25, 0x2f, 0x36, 0x39, 0x3c, 0x4a, 0x56, 0x58, 0x62, 0x67, 0x6e, 0x7d, 0x8a, 0x92, 0x96, 0x9b, 0x9d, 0xa5, 0xab, 0xae, 0xb5, 0xb9, 0xbd, 0xbf, 0xc1, 0xc8, 0xcc, 0xd1, 0xd7, 0xda, 0xe3, 0xe5, 0xed, 0xf1, 0xf3, 0xf6, 0xf9, 0xff, 0x10f, 0x11b, 0x11d, 0x123, 0x125, 0x127, 0x129, 0x12b, 0x12d, 0x12f, 0x131, 0x134, 0x137, 0x139, 0x13c, 0x13f, 0x143, 0x152, 0x15a, 0x15c, 0x15f, 0x161, 0x16a, 0x16e, 0x172, 0x174, 0x183, 0x187, 0x18d, 0x195, 0x199, 0x1a2, 0x1ab, 0x1b6, 0x1bc, 0x1c0, 0x1ce, 0x1dd, 0x1e1, 0x1e8, 0x1ed, 0x1fc, 0x208, 0x20b, 0x20d, 0x20f, 0x211, 0x213, 0x215, 0x217, 0x219, 0x21b, 0x21d, 0x220, 0x222, 0x224, 0x226, 0x228, 0x231, 0x233, 0x236, 0x239, 0x23c, 0x23e, 0x241, 0x243, 0x245, 0x247, 0x24a, 0x24c, 0x24e, 0x250, 0x252, 0x258, 0x25a, 0x25c, 0x25e, 0x260, 0x262, 0x26c, 0x26f, 0x271, 0x27b, 0x280, 0x282, 0x284, 0x286, 0x288, 0x28b, 0x28e, 0x292, 0x29a, 0x29c, 0x29e, 0x2a5, 0x2a7, 0x2ae, 0x2b6, 0x2bd, 0x2c3, 0x2c5, 0x2c7, 0x2ca, 0x2d3, 0x2d6, 0x2dd, 0x2e2, 0x2e5, 0x2e8, 0x2ec, 0x2ee, 0x2f0, 0x2f3, 0x2f6} + +// charInfoSparseValues: 760 entries, 3040 bytes +var charInfoSparseValues = [760]valueRange{ + // Block 0x0, offset 0x1 + {value: 0x0000, lo: 0x07}, + {value: 0x3000, lo: 0xa0, hi: 0xa0}, + {value: 0x3800, lo: 0xa8, hi: 0xa8}, + {value: 0x3000, lo: 0xaa, hi: 0xaa}, + {value: 0x3000, lo: 0xaf, hi: 0xaf}, + {value: 0x3000, lo: 0xb2, hi: 0xb5}, + {value: 0x3000, lo: 0xb8, hi: 0xba}, + {value: 0x3000, lo: 0xbc, hi: 0xbe}, + // Block 0x1, offset 0x2 + {value: 0x0000, lo: 0x0a}, + {value: 0x1100, lo: 0x80, hi: 0x81}, + {value: 0x9900, lo: 0x82, hi: 0x83}, + {value: 0x1100, lo: 0x84, hi: 0x8f}, + {value: 0x9900, lo: 0x92, hi: 0x93}, + {value: 0x1100, lo: 0x94, hi: 0xa5}, + {value: 0x1100, lo: 0xa8, hi: 0xb0}, + {value: 0x3000, lo: 0xb2, hi: 0xb3}, + {value: 0x1100, lo: 0xb4, hi: 0xb7}, + {value: 0x1100, lo: 0xb9, hi: 0xbe}, + {value: 0x3000, lo: 0xbf, hi: 0xbf}, + // Block 0x2, offset 0x3 + {value: 0x0000, lo: 0x0d}, + {value: 0x3000, lo: 0x80, hi: 0x80}, + {value: 0x1100, lo: 0x83, hi: 0x88}, + {value: 0x3000, lo: 0x89, hi: 0x89}, + {value: 0x9900, lo: 0x8c, hi: 0x8d}, + {value: 0x1100, lo: 0x8e, hi: 0x91}, + {value: 0x1100, lo: 0x94, hi: 0x99}, + {value: 0x9900, lo: 0x9a, hi: 0x9b}, + {value: 0x1100, lo: 0x9c, hi: 0x9f}, + {value: 0x9900, lo: 0xa0, hi: 0xa1}, + {value: 0x1100, lo: 0xa2, hi: 0xa5}, + {value: 0x9900, lo: 0xa8, hi: 0xab}, + {value: 0x1100, lo: 0xac, hi: 0xbe}, + {value: 0x3800, lo: 0xbf, hi: 0xbf}, + // Block 0x3, offset 0x4 + {value: 0x0000, lo: 0x03}, + {value: 0x9900, lo: 0xa0, hi: 0xa1}, + {value: 0x9900, lo: 0xaf, hi: 0xb0}, + {value: 0x8800, lo: 0xb7, hi: 0xb7}, + // Block 0x4, offset 0x5 + {value: 0x0000, lo: 0x09}, + {value: 0x3000, lo: 0x84, hi: 0x8c}, + {value: 0x1100, lo: 0x8d, hi: 0x9c}, + {value: 0x1100, lo: 0x9e, hi: 0xa3}, + {value: 0x1100, lo: 0xa6, hi: 0xa9}, + {value: 0x9900, lo: 0xaa, hi: 0xab}, + {value: 0x1100, lo: 0xac, hi: 0xb0}, + {value: 0x3000, lo: 0xb1, hi: 0xb3}, + {value: 0x1100, lo: 0xb4, hi: 0xb5}, + {value: 0x1100, lo: 0xb8, hi: 0xbf}, + // Block 0x5, offset 0x6 + {value: 0x0000, lo: 0x06}, + {value: 0x1100, lo: 0x80, hi: 0x9b}, + {value: 0x1100, lo: 0x9e, hi: 0x9f}, + {value: 0x9900, lo: 0xa6, hi: 0xa9}, + {value: 0x1100, lo: 0xaa, hi: 0xad}, + {value: 0x9900, lo: 0xae, hi: 0xaf}, + {value: 0x1100, lo: 0xb0, hi: 0xb3}, + // Block 0x6, offset 0x7 + {value: 0x0000, lo: 0x02}, + {value: 0x8800, lo: 0x92, hi: 0x92}, + {value: 0x3000, lo: 0xb0, hi: 0xb8}, + // Block 0x7, offset 0x8 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x98, hi: 0x9d}, + {value: 0x3000, lo: 0xa0, hi: 0xa4}, + // Block 0x8, offset 0x9 + {value: 0x0000, lo: 0x0d}, + {value: 0x8800, lo: 0x81, hi: 0x81}, + {value: 0x8800, lo: 0x85, hi: 0x85}, + {value: 0x8800, lo: 0x89, hi: 0x89}, + {value: 0x9900, lo: 0x8a, hi: 0x8b}, + {value: 0x1100, lo: 0x8c, hi: 0x8d}, + {value: 0x9900, lo: 0x8e, hi: 0x8e}, + {value: 0x3000, lo: 0x90, hi: 0x91}, + {value: 0x3800, lo: 0x92, hi: 0x92}, + {value: 0x3100, lo: 0x93, hi: 0x94}, + {value: 0x3000, lo: 0x95, hi: 0x96}, + {value: 0x3000, lo: 0xb0, hi: 0xb2}, + {value: 0x3000, lo: 0xb4, hi: 0xb5}, + {value: 0x3000, lo: 0xb9, hi: 0xb9}, + // Block 0x9, offset 0xa + {value: 0x0000, lo: 0x0b}, + {value: 0x8800, lo: 0x83, hi: 0x83}, + {value: 0x8800, lo: 0x87, hi: 0x87}, + {value: 0x8800, lo: 0x8b, hi: 0x8b}, + {value: 0x8800, lo: 0x8d, hi: 0x8d}, + {value: 0x1100, lo: 0x90, hi: 0x91}, + {value: 0x1100, lo: 0x93, hi: 0x93}, + {value: 0x8800, lo: 0x96, hi: 0x96}, + {value: 0x1100, lo: 0x97, hi: 0x97}, + {value: 0x1100, lo: 0x9c, hi: 0x9e}, + {value: 0x8800, lo: 0xb4, hi: 0xb5}, + {value: 0x1100, lo: 0xb6, hi: 0xb7}, + // Block 0xa, offset 0xb + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0x83, hi: 0x87}, + // Block 0xb, offset 0xc + {value: 0x0000, lo: 0x09}, + {value: 0x1100, lo: 0x81, hi: 0x82}, + {value: 0x1100, lo: 0x90, hi: 0x93}, + {value: 0x1100, lo: 0x96, hi: 0x97}, + {value: 0x8800, lo: 0x98, hi: 0x99}, + {value: 0x1100, lo: 0x9a, hi: 0x9f}, + {value: 0x1100, lo: 0xa2, hi: 0xa7}, + {value: 0x8800, lo: 0xa8, hi: 0xa9}, + {value: 0x1100, lo: 0xaa, hi: 0xb5}, + {value: 0x1100, lo: 0xb8, hi: 0xb9}, + // Block 0xc, offset 0xd + {value: 0x0001, lo: 0x04}, + {value: 0x0018, lo: 0x81, hi: 0x82}, + {value: 0x00e6, lo: 0x84, hi: 0x84}, + {value: 0x00dc, lo: 0x85, hi: 0x85}, + {value: 0x0012, lo: 0x87, hi: 0x87}, + // Block 0xd, offset 0xe + {value: 0x0000, lo: 0x06}, + {value: 0x00e6, lo: 0x90, hi: 0x97}, + {value: 0x001e, lo: 0x98, hi: 0x98}, + {value: 0x001f, lo: 0x99, hi: 0x99}, + {value: 0x0020, lo: 0x9a, hi: 0x9a}, + {value: 0x1100, lo: 0xa2, hi: 0xa6}, + {value: 0x8800, lo: 0xa7, hi: 0xa7}, + // Block 0xe, offset 0xf + {value: 0x0000, lo: 0x0e}, + {value: 0x1100, lo: 0x80, hi: 0x80}, + {value: 0x8800, lo: 0x81, hi: 0x81}, + {value: 0x1100, lo: 0x82, hi: 0x82}, + {value: 0x8800, lo: 0x92, hi: 0x92}, + {value: 0x1100, lo: 0x93, hi: 0x93}, + {value: 0x8800, lo: 0x95, hi: 0x95}, + {value: 0x00e6, lo: 0x96, hi: 0x9c}, + {value: 0x00e6, lo: 0x9f, hi: 0xa2}, + {value: 0x00dc, lo: 0xa3, hi: 0xa3}, + {value: 0x00e6, lo: 0xa4, hi: 0xa4}, + {value: 0x00e6, lo: 0xa7, hi: 0xa8}, + {value: 0x00dc, lo: 0xaa, hi: 0xaa}, + {value: 0x00e6, lo: 0xab, hi: 0xac}, + {value: 0x00dc, lo: 0xad, hi: 0xad}, + // Block 0xf, offset 0x10 + {value: 0x0000, lo: 0x0c}, + {value: 0x0024, lo: 0x91, hi: 0x91}, + {value: 0x00e6, lo: 0xb0, hi: 0xb0}, + {value: 0x00dc, lo: 0xb1, hi: 0xb1}, + {value: 0x00e6, lo: 0xb2, hi: 0xb3}, + {value: 0x00dc, lo: 0xb4, hi: 0xb4}, + {value: 0x00e6, lo: 0xb5, hi: 0xb6}, + {value: 0x00dc, lo: 0xb7, hi: 0xb9}, + {value: 0x00e6, lo: 0xba, hi: 0xba}, + {value: 0x00dc, lo: 0xbb, hi: 0xbc}, + {value: 0x00e6, lo: 0xbd, hi: 0xbd}, + {value: 0x00dc, lo: 0xbe, hi: 0xbe}, + {value: 0x00e6, lo: 0xbf, hi: 0xbf}, + // Block 0x10, offset 0x11 + {value: 0x000a, lo: 0x07}, + {value: 0x00e6, lo: 0x80, hi: 0x80}, + {value: 0x00e6, lo: 0x81, hi: 0x81}, + {value: 0x00dc, lo: 0x82, hi: 0x83}, + {value: 0x00dc, lo: 0x84, hi: 0x85}, + {value: 0x00dc, lo: 0x86, hi: 0x87}, + {value: 0x00dc, lo: 0x88, hi: 0x89}, + {value: 0x00e6, lo: 0x8a, hi: 0x8a}, + // Block 0x11, offset 0x12 + {value: 0x0000, lo: 0x03}, + {value: 0x00e6, lo: 0xab, hi: 0xb1}, + {value: 0x00dc, lo: 0xb2, hi: 0xb2}, + {value: 0x00e6, lo: 0xb3, hi: 0xb3}, + // Block 0x12, offset 0x13 + {value: 0x0000, lo: 0x04}, + {value: 0x00e6, lo: 0x96, hi: 0x99}, + {value: 0x00e6, lo: 0x9b, hi: 0xa3}, + {value: 0x00e6, lo: 0xa5, hi: 0xa7}, + {value: 0x00e6, lo: 0xa9, hi: 0xad}, + // Block 0x13, offset 0x14 + {value: 0x0000, lo: 0x01}, + {value: 0x00dc, lo: 0x99, hi: 0x9b}, + // Block 0x14, offset 0x15 + {value: 0x7700, lo: 0x07}, + {value: 0x8800, lo: 0xa8, hi: 0xa8}, + {value: 0x1100, lo: 0xa9, hi: 0xa9}, + {value: 0x8800, lo: 0xb0, hi: 0xb0}, + {value: 0x1100, lo: 0xb1, hi: 0xb1}, + {value: 0x8800, lo: 0xb3, hi: 0xb3}, + {value: 0x1100, lo: 0xb4, hi: 0xb4}, + {value: 0x6607, lo: 0xbc, hi: 0xbc}, + // Block 0x15, offset 0x16 + {value: 0x0000, lo: 0x05}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x00e6, lo: 0x91, hi: 0x91}, + {value: 0x00dc, lo: 0x92, hi: 0x92}, + {value: 0x00e6, lo: 0x93, hi: 0x94}, + {value: 0x3300, lo: 0x98, hi: 0x9f}, + // Block 0x16, offset 0x17 + {value: 0x65f9, lo: 0x02}, + {value: 0x0007, lo: 0xbc, hi: 0xbc}, + {value: 0x6600, lo: 0xbe, hi: 0xbe}, + // Block 0x17, offset 0x18 + {value: 0x0000, lo: 0x06}, + {value: 0x8800, lo: 0x87, hi: 0x87}, + {value: 0x1100, lo: 0x8b, hi: 0x8c}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x6600, lo: 0x97, hi: 0x97}, + {value: 0x3300, lo: 0x9c, hi: 0x9d}, + {value: 0x3300, lo: 0x9f, hi: 0x9f}, + // Block 0x18, offset 0x19 + {value: 0x0000, lo: 0x03}, + {value: 0x3300, lo: 0xb3, hi: 0xb3}, + {value: 0x3300, lo: 0xb6, hi: 0xb6}, + {value: 0x0007, lo: 0xbc, hi: 0xbc}, + // Block 0x19, offset 0x1a + {value: 0x0000, lo: 0x03}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x3300, lo: 0x99, hi: 0x9b}, + {value: 0x3300, lo: 0x9e, hi: 0x9e}, + // Block 0x1a, offset 0x1b + {value: 0x0000, lo: 0x01}, + {value: 0x0007, lo: 0xbc, hi: 0xbc}, + // Block 0x1b, offset 0x1c + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + // Block 0x1c, offset 0x1d + {value: 0x0000, lo: 0x06}, + {value: 0x8800, lo: 0x87, hi: 0x87}, + {value: 0x1100, lo: 0x88, hi: 0x88}, + {value: 0x1100, lo: 0x8b, hi: 0x8c}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x6600, lo: 0x96, hi: 0x97}, + {value: 0x3300, lo: 0x9c, hi: 0x9d}, + // Block 0x1d, offset 0x1e + {value: 0x5500, lo: 0x03}, + {value: 0x8800, lo: 0x92, hi: 0x92}, + {value: 0x1100, lo: 0x94, hi: 0x94}, + {value: 0x6600, lo: 0xbe, hi: 0xbe}, + // Block 0x1e, offset 0x1f + {value: 0x0000, lo: 0x04}, + {value: 0x8800, lo: 0x86, hi: 0x87}, + {value: 0x1100, lo: 0x8a, hi: 0x8c}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x6600, lo: 0x97, hi: 0x97}, + // Block 0x1f, offset 0x20 + {value: 0x004b, lo: 0x05}, + {value: 0x8800, lo: 0x86, hi: 0x86}, + {value: 0x1100, lo: 0x88, hi: 0x88}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x0054, lo: 0x95, hi: 0x95}, + {value: 0x665b, lo: 0x96, hi: 0x96}, + // Block 0x20, offset 0x21 + {value: 0x87f9, lo: 0x02}, + {value: 0x0007, lo: 0xbc, hi: 0xbc}, + {value: 0x8800, lo: 0xbf, hi: 0xbf}, + // Block 0x21, offset 0x22 + {value: 0x0000, lo: 0x08}, + {value: 0x1100, lo: 0x80, hi: 0x80}, + {value: 0x6600, lo: 0x82, hi: 0x82}, + {value: 0x8800, lo: 0x86, hi: 0x86}, + {value: 0x1100, lo: 0x87, hi: 0x88}, + {value: 0x9900, lo: 0x8a, hi: 0x8a}, + {value: 0x1100, lo: 0x8b, hi: 0x8b}, + {value: 0x0009, lo: 0x8d, hi: 0x8d}, + {value: 0x6600, lo: 0x95, hi: 0x96}, + // Block 0x22, offset 0x23 + {value: 0x0000, lo: 0x01}, + {value: 0x6600, lo: 0xbe, hi: 0xbe}, + // Block 0x23, offset 0x24 + {value: 0x0000, lo: 0x07}, + {value: 0x6609, lo: 0x8a, hi: 0x8a}, + {value: 0x6600, lo: 0x8f, hi: 0x8f}, + {value: 0x8800, lo: 0x99, hi: 0x99}, + {value: 0x1100, lo: 0x9a, hi: 0x9a}, + {value: 0x9900, lo: 0x9c, hi: 0x9c}, + {value: 0x1100, lo: 0x9d, hi: 0x9e}, + {value: 0x6600, lo: 0x9f, hi: 0x9f}, + // Block 0x24, offset 0x25 + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0xb3, hi: 0xb3}, + {value: 0x0067, lo: 0xb8, hi: 0xb9}, + {value: 0x0009, lo: 0xba, hi: 0xba}, + // Block 0x25, offset 0x26 + {value: 0x0000, lo: 0x01}, + {value: 0x006b, lo: 0x88, hi: 0x8b}, + // Block 0x26, offset 0x27 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0xb3, hi: 0xb3}, + {value: 0x0076, lo: 0xb8, hi: 0xb9}, + // Block 0x27, offset 0x28 + {value: 0x0000, lo: 0x02}, + {value: 0x007a, lo: 0x88, hi: 0x8b}, + {value: 0x3000, lo: 0x9c, hi: 0x9d}, + // Block 0x28, offset 0x29 + {value: 0x0000, lo: 0x05}, + {value: 0x3000, lo: 0x8c, hi: 0x8c}, + {value: 0x00dc, lo: 0x98, hi: 0x99}, + {value: 0x00dc, lo: 0xb5, hi: 0xb5}, + {value: 0x00dc, lo: 0xb7, hi: 0xb7}, + {value: 0x00d8, lo: 0xb9, hi: 0xb9}, + // Block 0x29, offset 0x2a + {value: 0x0000, lo: 0x0f}, + {value: 0x3300, lo: 0x83, hi: 0x83}, + {value: 0x3300, lo: 0x8d, hi: 0x8d}, + {value: 0x3300, lo: 0x92, hi: 0x92}, + {value: 0x3300, lo: 0x97, hi: 0x97}, + {value: 0x3300, lo: 0x9c, hi: 0x9c}, + {value: 0x3300, lo: 0xa9, hi: 0xa9}, + {value: 0x0081, lo: 0xb1, hi: 0xb1}, + {value: 0x0082, lo: 0xb2, hi: 0xb2}, + {value: 0x3300, lo: 0xb3, hi: 0xb3}, + {value: 0x0084, lo: 0xb4, hi: 0xb4}, + {value: 0x3300, lo: 0xb5, hi: 0xb6}, + {value: 0x3000, lo: 0xb7, hi: 0xb7}, + {value: 0x3300, lo: 0xb8, hi: 0xb8}, + {value: 0x3000, lo: 0xb9, hi: 0xb9}, + {value: 0x0082, lo: 0xba, hi: 0xbd}, + // Block 0x2a, offset 0x2b + {value: 0x0000, lo: 0x0b}, + {value: 0x0082, lo: 0x80, hi: 0x80}, + {value: 0x3300, lo: 0x81, hi: 0x81}, + {value: 0x00e6, lo: 0x82, hi: 0x83}, + {value: 0x0009, lo: 0x84, hi: 0x84}, + {value: 0x00e6, lo: 0x86, hi: 0x87}, + {value: 0x3300, lo: 0x93, hi: 0x93}, + {value: 0x3300, lo: 0x9d, hi: 0x9d}, + {value: 0x3300, lo: 0xa2, hi: 0xa2}, + {value: 0x3300, lo: 0xa7, hi: 0xa7}, + {value: 0x3300, lo: 0xac, hi: 0xac}, + {value: 0x3300, lo: 0xb9, hi: 0xb9}, + // Block 0x2b, offset 0x2c + {value: 0x0000, lo: 0x01}, + {value: 0x00dc, lo: 0x86, hi: 0x86}, + // Block 0x2c, offset 0x2d + {value: 0x0000, lo: 0x05}, + {value: 0x8800, lo: 0xa5, hi: 0xa5}, + {value: 0x1100, lo: 0xa6, hi: 0xa6}, + {value: 0x6600, lo: 0xae, hi: 0xae}, + {value: 0x0007, lo: 0xb7, hi: 0xb7}, + {value: 0x0009, lo: 0xb9, hi: 0xba}, + // Block 0x2d, offset 0x2e + {value: 0x0000, lo: 0x01}, + {value: 0x00dc, lo: 0x8d, hi: 0x8d}, + // Block 0x2e, offset 0x2f + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xbc, hi: 0xbc}, + // Block 0x2f, offset 0x30 + {value: 0x0000, lo: 0x01}, + {value: 0x8800, lo: 0x80, hi: 0x92}, + // Block 0x30, offset 0x31 + {value: 0x0000, lo: 0x01}, + {value: 0xee00, lo: 0xa1, hi: 0xb5}, + // Block 0x31, offset 0x32 + {value: 0x0000, lo: 0x01}, + {value: 0x6600, lo: 0xa8, hi: 0xbf}, + // Block 0x32, offset 0x33 + {value: 0x0000, lo: 0x01}, + {value: 0x6600, lo: 0x80, hi: 0x82}, + // Block 0x33, offset 0x34 + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0x9d, hi: 0x9f}, + // Block 0x34, offset 0x35 + {value: 0x0000, lo: 0x02}, + {value: 0x0009, lo: 0x94, hi: 0x94}, + {value: 0x0009, lo: 0xb4, hi: 0xb4}, + // Block 0x35, offset 0x36 + {value: 0x00dd, lo: 0x02}, + {value: 0x0009, lo: 0x92, hi: 0x92}, + {value: 0x00e6, lo: 0x9d, hi: 0x9d}, + // Block 0x36, offset 0x37 + {value: 0x0000, lo: 0x01}, + {value: 0x00e4, lo: 0xa9, hi: 0xa9}, + // Block 0x37, offset 0x38 + {value: 0x0008, lo: 0x02}, + {value: 0x00de, lo: 0xb9, hi: 0xba}, + {value: 0x00dc, lo: 0xbb, hi: 0xbb}, + // Block 0x38, offset 0x39 + {value: 0x0000, lo: 0x02}, + {value: 0x00e6, lo: 0x97, hi: 0x97}, + {value: 0x00dc, lo: 0x98, hi: 0x98}, + // Block 0x39, offset 0x3a + {value: 0x0000, lo: 0x03}, + {value: 0x0009, lo: 0xa0, hi: 0xa0}, + {value: 0x00e6, lo: 0xb5, hi: 0xbc}, + {value: 0x00dc, lo: 0xbf, hi: 0xbf}, + // Block 0x3a, offset 0x3b + {value: 0x7700, lo: 0x0e}, + {value: 0x8800, lo: 0x85, hi: 0x85}, + {value: 0x1100, lo: 0x86, hi: 0x87}, + {value: 0x1100, lo: 0x88, hi: 0x89}, + {value: 0x1100, lo: 0x8a, hi: 0x8b}, + {value: 0x1100, lo: 0x8c, hi: 0x8d}, + {value: 0x1100, lo: 0x8e, hi: 0x8e}, + {value: 0x8800, lo: 0x91, hi: 0x91}, + {value: 0x1100, lo: 0x92, hi: 0x92}, + {value: 0x0007, lo: 0xb4, hi: 0xb4}, + {value: 0x6600, lo: 0xb5, hi: 0xb5}, + {value: 0x8800, lo: 0xba, hi: 0xba}, + {value: 0x1100, lo: 0xbb, hi: 0xbc}, + {value: 0x1100, lo: 0xbd, hi: 0xbe}, + {value: 0x8800, lo: 0xbf, hi: 0xbf}, + // Block 0x3b, offset 0x3c + {value: 0x0000, lo: 0x07}, + {value: 0x1100, lo: 0x80, hi: 0x81}, + {value: 0x8800, lo: 0x82, hi: 0x82}, + {value: 0x1100, lo: 0x83, hi: 0x83}, + {value: 0x0009, lo: 0x84, hi: 0x84}, + {value: 0x00e6, lo: 0xab, hi: 0xab}, + {value: 0x00dc, lo: 0xac, hi: 0xac}, + {value: 0x00e6, lo: 0xad, hi: 0xb3}, + // Block 0x3c, offset 0x3d + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0xaa, hi: 0xaa}, + // Block 0x3d, offset 0x3e + {value: 0x0000, lo: 0x02}, + {value: 0x0007, lo: 0xa6, hi: 0xa6}, + {value: 0x0009, lo: 0xb2, hi: 0xb3}, + // Block 0x3e, offset 0x3f + {value: 0x0000, lo: 0x01}, + {value: 0x0007, lo: 0xb7, hi: 0xb7}, + // Block 0x3f, offset 0x40 + {value: 0x0000, lo: 0x08}, + {value: 0x00e6, lo: 0x90, hi: 0x92}, + {value: 0x0001, lo: 0x94, hi: 0x94}, + {value: 0x00dc, lo: 0x95, hi: 0x99}, + {value: 0x00e6, lo: 0x9a, hi: 0x9b}, + {value: 0x00dc, lo: 0x9c, hi: 0x9f}, + {value: 0x00e6, lo: 0xa0, hi: 0xa0}, + {value: 0x0001, lo: 0xa2, hi: 0xa8}, + {value: 0x00dc, lo: 0xad, hi: 0xad}, + // Block 0x40, offset 0x41 + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0xac, hi: 0xae}, + {value: 0x3000, lo: 0xb0, hi: 0xba}, + {value: 0x3000, lo: 0xbc, hi: 0xbf}, + // Block 0x41, offset 0x42 + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0x80, hi: 0x8d}, + {value: 0x3000, lo: 0x8f, hi: 0xaa}, + {value: 0x3000, lo: 0xb8, hi: 0xb8}, + // Block 0x42, offset 0x43 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x9b, hi: 0xbf}, + // Block 0x43, offset 0x44 + {value: 0x0000, lo: 0x0e}, + {value: 0x00e6, lo: 0x80, hi: 0x81}, + {value: 0x00dc, lo: 0x82, hi: 0x82}, + {value: 0x00e6, lo: 0x83, hi: 0x89}, + {value: 0x00dc, lo: 0x8a, hi: 0x8a}, + {value: 0x00e6, lo: 0x8b, hi: 0x8c}, + {value: 0x00ea, lo: 0x8d, hi: 0x8d}, + {value: 0x00d6, lo: 0x8e, hi: 0x8e}, + {value: 0x00dc, lo: 0x8f, hi: 0x8f}, + {value: 0x00ca, lo: 0x90, hi: 0x90}, + {value: 0x00e6, lo: 0x91, hi: 0xa6}, + {value: 0x00e9, lo: 0xbc, hi: 0xbc}, + {value: 0x00dc, lo: 0xbd, hi: 0xbd}, + {value: 0x00e6, lo: 0xbe, hi: 0xbe}, + {value: 0x00dc, lo: 0xbf, hi: 0xbf}, + // Block 0x44, offset 0x45 + {value: 0x0000, lo: 0x03}, + {value: 0x1100, lo: 0x80, hi: 0xb5}, + {value: 0x9900, lo: 0xb6, hi: 0xb7}, + {value: 0x1100, lo: 0xb8, hi: 0xbf}, + // Block 0x45, offset 0x46 + {value: 0x0000, lo: 0x05}, + {value: 0x1100, lo: 0x80, hi: 0x99}, + {value: 0x9900, lo: 0x9a, hi: 0x9b}, + {value: 0x1100, lo: 0x9c, hi: 0xa1}, + {value: 0x9900, lo: 0xa2, hi: 0xa3}, + {value: 0x1100, lo: 0xa4, hi: 0xbf}, + // Block 0x46, offset 0x47 + {value: 0x0000, lo: 0x07}, + {value: 0x1100, lo: 0x80, hi: 0x99}, + {value: 0x3000, lo: 0x9a, hi: 0x9a}, + {value: 0x3100, lo: 0x9b, hi: 0x9b}, + {value: 0x9900, lo: 0xa0, hi: 0xa1}, + {value: 0x1100, lo: 0xa2, hi: 0xb7}, + {value: 0x9900, lo: 0xb8, hi: 0xb9}, + {value: 0x1100, lo: 0xba, hi: 0xbf}, + // Block 0x47, offset 0x48 + {value: 0x0000, lo: 0x03}, + {value: 0x1100, lo: 0x80, hi: 0x8b}, + {value: 0x9900, lo: 0x8c, hi: 0x8d}, + {value: 0x1100, lo: 0x8e, hi: 0xb9}, + // Block 0x48, offset 0x49 + {value: 0x0000, lo: 0x08}, + {value: 0x9900, lo: 0x80, hi: 0x91}, + {value: 0x1100, lo: 0x92, hi: 0x95}, + {value: 0x9900, lo: 0x98, hi: 0x99}, + {value: 0x1100, lo: 0x9a, hi: 0x9d}, + {value: 0x9900, lo: 0xa0, hi: 0xb1}, + {value: 0x1100, lo: 0xb2, hi: 0xb7}, + {value: 0x9900, lo: 0xb8, hi: 0xb9}, + {value: 0x1100, lo: 0xba, hi: 0xbf}, + // Block 0x49, offset 0x4a + {value: 0x0000, lo: 0x08}, + {value: 0x1100, lo: 0x80, hi: 0xb4}, + {value: 0x9900, lo: 0xb6, hi: 0xb6}, + {value: 0x1100, lo: 0xb7, hi: 0xba}, + {value: 0x3300, lo: 0xbb, hi: 0xbb}, + {value: 0x1100, lo: 0xbc, hi: 0xbc}, + {value: 0x3000, lo: 0xbd, hi: 0xbd}, + {value: 0x3300, lo: 0xbe, hi: 0xbe}, + {value: 0x3800, lo: 0xbf, hi: 0xbf}, + // Block 0x4a, offset 0x4b + {value: 0x0000, lo: 0x0a}, + {value: 0x3300, lo: 0x80, hi: 0x81}, + {value: 0x3000, lo: 0x82, hi: 0x8a}, + {value: 0x3000, lo: 0x91, hi: 0x91}, + {value: 0x3000, lo: 0x97, hi: 0x97}, + {value: 0x3000, lo: 0xa4, hi: 0xa6}, + {value: 0x3000, lo: 0xaf, hi: 0xaf}, + {value: 0x3000, lo: 0xb3, hi: 0xb4}, + {value: 0x3000, lo: 0xb6, hi: 0xb7}, + {value: 0x3000, lo: 0xbc, hi: 0xbc}, + {value: 0x3000, lo: 0xbe, hi: 0xbe}, + // Block 0x4b, offset 0x4c + {value: 0x0000, lo: 0x05}, + {value: 0x3000, lo: 0x87, hi: 0x89}, + {value: 0x3000, lo: 0x97, hi: 0x97}, + {value: 0x3000, lo: 0x9f, hi: 0x9f}, + {value: 0x3000, lo: 0xb0, hi: 0xb1}, + {value: 0x3000, lo: 0xb4, hi: 0xbf}, + // Block 0x4c, offset 0x4d + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0x80, hi: 0x8e}, + {value: 0x3000, lo: 0x90, hi: 0x9c}, + {value: 0x3000, lo: 0xa8, hi: 0xa8}, + // Block 0x4d, offset 0x4e + {value: 0x0000, lo: 0x0d}, + {value: 0x00e6, lo: 0x90, hi: 0x91}, + {value: 0x0001, lo: 0x92, hi: 0x93}, + {value: 0x00e6, lo: 0x94, hi: 0x97}, + {value: 0x0001, lo: 0x98, hi: 0x9a}, + {value: 0x00e6, lo: 0x9b, hi: 0x9c}, + {value: 0x00e6, lo: 0xa1, hi: 0xa1}, + {value: 0x0001, lo: 0xa5, hi: 0xa6}, + {value: 0x00e6, lo: 0xa7, hi: 0xa7}, + {value: 0x00dc, lo: 0xa8, hi: 0xa8}, + {value: 0x00e6, lo: 0xa9, hi: 0xa9}, + {value: 0x0001, lo: 0xaa, hi: 0xab}, + {value: 0x00dc, lo: 0xac, hi: 0xaf}, + {value: 0x00e6, lo: 0xb0, hi: 0xb0}, + // Block 0x4e, offset 0x4f + {value: 0x0000, lo: 0x0e}, + {value: 0x3000, lo: 0x80, hi: 0x83}, + {value: 0x3000, lo: 0x85, hi: 0x87}, + {value: 0x3000, lo: 0x89, hi: 0x93}, + {value: 0x3000, lo: 0x95, hi: 0x96}, + {value: 0x3000, lo: 0x99, hi: 0x9d}, + {value: 0x3000, lo: 0xa0, hi: 0xa2}, + {value: 0x3000, lo: 0xa4, hi: 0xa4}, + {value: 0x3300, lo: 0xa6, hi: 0xa6}, + {value: 0x3000, lo: 0xa8, hi: 0xa8}, + {value: 0x3300, lo: 0xaa, hi: 0xab}, + {value: 0x3000, lo: 0xac, hi: 0xad}, + {value: 0x3000, lo: 0xaf, hi: 0xb1}, + {value: 0x3000, lo: 0xb3, hi: 0xb9}, + {value: 0x3000, lo: 0xbb, hi: 0xbf}, + // Block 0x4f, offset 0x50 + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0x80, hi: 0x80}, + {value: 0x3000, lo: 0x85, hi: 0x89}, + {value: 0x3000, lo: 0x90, hi: 0xbf}, + // Block 0x50, offset 0x51 + {value: 0x0000, lo: 0x06}, + {value: 0x3000, lo: 0x89, hi: 0x89}, + {value: 0x8800, lo: 0x90, hi: 0x90}, + {value: 0x8800, lo: 0x92, hi: 0x92}, + {value: 0x8800, lo: 0x94, hi: 0x94}, + {value: 0x1100, lo: 0x9a, hi: 0x9b}, + {value: 0x1100, lo: 0xae, hi: 0xae}, + // Block 0x51, offset 0x52 + {value: 0x0000, lo: 0x04}, + {value: 0x1100, lo: 0x8d, hi: 0x8f}, + {value: 0x8800, lo: 0x90, hi: 0x90}, + {value: 0x8800, lo: 0x92, hi: 0x92}, + {value: 0x8800, lo: 0x94, hi: 0x94}, + // Block 0x52, offset 0x53 + {value: 0x7700, lo: 0x0e}, + {value: 0x8800, lo: 0x83, hi: 0x83}, + {value: 0x1100, lo: 0x84, hi: 0x84}, + {value: 0x8800, lo: 0x88, hi: 0x88}, + {value: 0x1100, lo: 0x89, hi: 0x89}, + {value: 0x8800, lo: 0x8b, hi: 0x8b}, + {value: 0x1100, lo: 0x8c, hi: 0x8c}, + {value: 0x8800, lo: 0xa3, hi: 0xa3}, + {value: 0x1100, lo: 0xa4, hi: 0xa5}, + {value: 0x1100, lo: 0xa6, hi: 0xa6}, + {value: 0x3000, lo: 0xac, hi: 0xac}, + {value: 0x3000, lo: 0xad, hi: 0xad}, + {value: 0x3000, lo: 0xaf, hi: 0xaf}, + {value: 0x3000, lo: 0xb0, hi: 0xb0}, + {value: 0x8800, lo: 0xbc, hi: 0xbc}, + // Block 0x53, offset 0x54 + {value: 0x0000, lo: 0x0b}, + {value: 0x1100, lo: 0x80, hi: 0x81}, + {value: 0x8800, lo: 0x82, hi: 0x83}, + {value: 0x1100, lo: 0x84, hi: 0x85}, + {value: 0x8800, lo: 0x86, hi: 0x87}, + {value: 0x1100, lo: 0x88, hi: 0x89}, + {value: 0x8800, lo: 0x91, hi: 0x92}, + {value: 0x8800, lo: 0xa2, hi: 0xa2}, + {value: 0x8800, lo: 0xa8, hi: 0xa9}, + {value: 0x8800, lo: 0xab, hi: 0xab}, + {value: 0x1100, lo: 0xac, hi: 0xaf}, + {value: 0x8800, lo: 0xb2, hi: 0xb5}, + // Block 0x54, offset 0x55 + {value: 0x0000, lo: 0x02}, + {value: 0x1100, lo: 0xa0, hi: 0xa3}, + {value: 0x1100, lo: 0xaa, hi: 0xad}, + // Block 0x55, offset 0x56 + {value: 0x0000, lo: 0x01}, + {value: 0x3300, lo: 0xa9, hi: 0xaa}, + // Block 0x56, offset 0x57 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xa0, hi: 0xbf}, + // Block 0x57, offset 0x58 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0xbf}, + // Block 0x58, offset 0x59 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0xaa}, + // Block 0x59, offset 0x5a + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x8c, hi: 0x8c}, + // Block 0x5a, offset 0x5b + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xb4, hi: 0xb6}, + // Block 0x5b, offset 0x5c + {value: 0x0000, lo: 0x01}, + {value: 0x3300, lo: 0x9c, hi: 0x9c}, + // Block 0x5c, offset 0x5d + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xbc, hi: 0xbd}, + // Block 0x5d, offset 0x5e + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0xaf, hi: 0xb1}, + // Block 0x5e, offset 0x5f + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0xaf, hi: 0xaf}, + {value: 0x0009, lo: 0xbf, hi: 0xbf}, + // Block 0x5f, offset 0x60 + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0xa0, hi: 0xbf}, + // Block 0x60, offset 0x61 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x9f, hi: 0x9f}, + // Block 0x61, offset 0x62 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xb3, hi: 0xb3}, + // Block 0x62, offset 0x63 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0x95}, + // Block 0x63, offset 0x64 + {value: 0x0000, lo: 0x08}, + {value: 0x3000, lo: 0x80, hi: 0x80}, + {value: 0x00da, lo: 0xaa, hi: 0xaa}, + {value: 0x00e4, lo: 0xab, hi: 0xab}, + {value: 0x00e8, lo: 0xac, hi: 0xac}, + {value: 0x00de, lo: 0xad, hi: 0xad}, + {value: 0x00e0, lo: 0xae, hi: 0xaf}, + {value: 0x3000, lo: 0xb6, hi: 0xb6}, + {value: 0x3000, lo: 0xb8, hi: 0xba}, + // Block 0x64, offset 0x65 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xb1, hi: 0xbf}, + // Block 0x65, offset 0x66 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x8e}, + {value: 0x3000, lo: 0x92, hi: 0x9f}, + // Block 0x66, offset 0x67 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x9e}, + {value: 0x3000, lo: 0xa0, hi: 0xbf}, + // Block 0x67, offset 0x68 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x87}, + {value: 0x3000, lo: 0x90, hi: 0xbe}, + // Block 0x68, offset 0x69 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0xbe}, + // Block 0x69, offset 0x6a + {value: 0x0000, lo: 0x02}, + {value: 0x00e6, lo: 0xaf, hi: 0xaf}, + {value: 0x00e6, lo: 0xbc, hi: 0xbd}, + // Block 0x6a, offset 0x6b + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0xb0, hi: 0xb1}, + // Block 0x6b, offset 0x6c + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0xb0, hi: 0xb0}, + // Block 0x6c, offset 0x6d + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0x86, hi: 0x86}, + // Block 0x6d, offset 0x6e + {value: 0x0000, lo: 0x02}, + {value: 0x0009, lo: 0x84, hi: 0x84}, + {value: 0x00e6, lo: 0xa0, hi: 0xb1}, + // Block 0x6e, offset 0x6f + {value: 0x0000, lo: 0x01}, + {value: 0x00dc, lo: 0xab, hi: 0xad}, + // Block 0x6f, offset 0x70 + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0x93, hi: 0x93}, + // Block 0x70, offset 0x71 + {value: 0x0000, lo: 0x01}, + {value: 0x0007, lo: 0xb3, hi: 0xb3}, + // Block 0x71, offset 0x72 + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0x80, hi: 0x80}, + // Block 0x72, offset 0x73 + {value: 0x0000, lo: 0x05}, + {value: 0x00e6, lo: 0xb0, hi: 0xb0}, + {value: 0x00e6, lo: 0xb2, hi: 0xb3}, + {value: 0x00dc, lo: 0xb4, hi: 0xb4}, + {value: 0x00e6, lo: 0xb7, hi: 0xb8}, + {value: 0x00e6, lo: 0xbe, hi: 0xbf}, + // Block 0x73, offset 0x74 + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0x81, hi: 0x81}, + // Block 0x74, offset 0x75 + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0xad, hi: 0xad}, + // Block 0x75, offset 0x76 + {value: 0x0000, lo: 0x01}, + {value: 0x1100, lo: 0x80, hi: 0xbf}, + // Block 0x76, offset 0x77 + {value: 0x0000, lo: 0x01}, + {value: 0x1100, lo: 0x80, hi: 0xa3}, + // Block 0x77, offset 0x78 + {value: 0x0000, lo: 0x01}, + {value: 0x3300, lo: 0x80, hi: 0xbf}, + // Block 0x78, offset 0x79 + {value: 0x0000, lo: 0x09}, + {value: 0x3300, lo: 0x80, hi: 0x8d}, + {value: 0x3300, lo: 0x90, hi: 0x90}, + {value: 0x3300, lo: 0x92, hi: 0x92}, + {value: 0x3300, lo: 0x95, hi: 0x9e}, + {value: 0x3300, lo: 0xa0, hi: 0xa0}, + {value: 0x3300, lo: 0xa2, hi: 0xa2}, + {value: 0x3300, lo: 0xa5, hi: 0xa6}, + {value: 0x3300, lo: 0xaa, hi: 0xad}, + {value: 0x3300, lo: 0xb0, hi: 0xbf}, + // Block 0x79, offset 0x7a + {value: 0x0000, lo: 0x02}, + {value: 0x3300, lo: 0x80, hi: 0xad}, + {value: 0x3300, lo: 0xb0, hi: 0xbf}, + // Block 0x7a, offset 0x7b + {value: 0x0000, lo: 0x01}, + {value: 0x3300, lo: 0x80, hi: 0x99}, + // Block 0x7b, offset 0x7c + {value: 0x0000, lo: 0x09}, + {value: 0x3000, lo: 0x80, hi: 0x86}, + {value: 0x3000, lo: 0x93, hi: 0x97}, + {value: 0x3300, lo: 0x9d, hi: 0x9d}, + {value: 0x001a, lo: 0x9e, hi: 0x9e}, + {value: 0x3300, lo: 0x9f, hi: 0x9f}, + {value: 0x3000, lo: 0xa0, hi: 0xa9}, + {value: 0x3300, lo: 0xaa, hi: 0xb6}, + {value: 0x3300, lo: 0xb8, hi: 0xbc}, + {value: 0x3300, lo: 0xbe, hi: 0xbe}, + // Block 0x7c, offset 0x7d + {value: 0x0000, lo: 0x04}, + {value: 0x3300, lo: 0x80, hi: 0x81}, + {value: 0x3300, lo: 0x83, hi: 0x84}, + {value: 0x3300, lo: 0x86, hi: 0x8e}, + {value: 0x3000, lo: 0x8f, hi: 0xbf}, + // Block 0x7d, offset 0x7e + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0xb1}, + // Block 0x7e, offset 0x7f + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x93, hi: 0xbf}, + // Block 0x7f, offset 0x80 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0xbd}, + // Block 0x80, offset 0x81 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x90, hi: 0xbf}, + // Block 0x81, offset 0x82 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x8f}, + {value: 0x3000, lo: 0x92, hi: 0xbf}, + // Block 0x82, offset 0x83 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x87}, + {value: 0x3000, lo: 0xb0, hi: 0xbc}, + // Block 0x83, offset 0x84 + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0x90, hi: 0x99}, + {value: 0x00e6, lo: 0xa0, hi: 0xa6}, + {value: 0x3000, lo: 0xb0, hi: 0xbf}, + // Block 0x84, offset 0x85 + {value: 0x0000, lo: 0x07}, + {value: 0x3000, lo: 0x80, hi: 0x84}, + {value: 0x3000, lo: 0x87, hi: 0x92}, + {value: 0x3000, lo: 0x94, hi: 0xa6}, + {value: 0x3000, lo: 0xa8, hi: 0xab}, + {value: 0x3000, lo: 0xb0, hi: 0xb2}, + {value: 0x3000, lo: 0xb4, hi: 0xb4}, + {value: 0x3000, lo: 0xb6, hi: 0xbf}, + // Block 0x85, offset 0x86 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0xbc}, + // Block 0x86, offset 0x87 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x81, hi: 0xbf}, + // Block 0x87, offset 0x88 + {value: 0x0000, lo: 0x06}, + {value: 0x3000, lo: 0x82, hi: 0x87}, + {value: 0x3000, lo: 0x8a, hi: 0x8f}, + {value: 0x3000, lo: 0x92, hi: 0x97}, + {value: 0x3000, lo: 0x9a, hi: 0x9c}, + {value: 0x3000, lo: 0xa0, hi: 0xa6}, + {value: 0x3000, lo: 0xa8, hi: 0xae}, + // Block 0x88, offset 0x89 + {value: 0x0000, lo: 0x01}, + {value: 0x00dc, lo: 0xbd, hi: 0xbd}, + // Block 0x89, offset 0x8a + {value: 0x0000, lo: 0x06}, + {value: 0x00dc, lo: 0x8d, hi: 0x8d}, + {value: 0x00e6, lo: 0x8f, hi: 0x8f}, + {value: 0x00e6, lo: 0xb8, hi: 0xb8}, + {value: 0x0001, lo: 0xb9, hi: 0xb9}, + {value: 0x00dc, lo: 0xba, hi: 0xba}, + {value: 0x0009, lo: 0xbf, hi: 0xbf}, + // Block 0x8a, offset 0x8b + {value: 0x7700, lo: 0x07}, + {value: 0x8800, lo: 0x99, hi: 0x99}, + {value: 0x1100, lo: 0x9a, hi: 0x9b}, + {value: 0x1100, lo: 0x9c, hi: 0x9c}, + {value: 0x8800, lo: 0xa5, hi: 0xa5}, + {value: 0x1100, lo: 0xab, hi: 0xab}, + {value: 0x0009, lo: 0xb9, hi: 0xb9}, + {value: 0x6607, lo: 0xba, hi: 0xba}, + // Block 0x8b, offset 0x8c + {value: 0x0000, lo: 0x06}, + {value: 0x3300, lo: 0x9e, hi: 0xa4}, + {value: 0x00d8, lo: 0xa5, hi: 0xa6}, + {value: 0x0001, lo: 0xa7, hi: 0xa9}, + {value: 0x00e2, lo: 0xad, hi: 0xad}, + {value: 0x00d8, lo: 0xae, hi: 0xb2}, + {value: 0x00dc, lo: 0xbb, hi: 0xbf}, + // Block 0x8c, offset 0x8d + {value: 0x0000, lo: 0x05}, + {value: 0x00dc, lo: 0x80, hi: 0x82}, + {value: 0x00e6, lo: 0x85, hi: 0x89}, + {value: 0x00dc, lo: 0x8a, hi: 0x8b}, + {value: 0x00e6, lo: 0xaa, hi: 0xad}, + {value: 0x3300, lo: 0xbb, hi: 0xbf}, + // Block 0x8d, offset 0x8e + {value: 0x0000, lo: 0x01}, + {value: 0x3300, lo: 0x80, hi: 0x80}, + // Block 0x8e, offset 0x8f + {value: 0x0000, lo: 0x01}, + {value: 0x00e6, lo: 0x82, hi: 0x84}, + // Block 0x8f, offset 0x90 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x94}, + {value: 0x3000, lo: 0x96, hi: 0xbf}, + // Block 0x90, offset 0x91 + {value: 0x0000, lo: 0x08}, + {value: 0x3000, lo: 0x80, hi: 0x9c}, + {value: 0x3000, lo: 0x9e, hi: 0x9f}, + {value: 0x3000, lo: 0xa2, hi: 0xa2}, + {value: 0x3000, lo: 0xa5, hi: 0xa6}, + {value: 0x3000, lo: 0xa9, hi: 0xac}, + {value: 0x3000, lo: 0xae, hi: 0xb9}, + {value: 0x3000, lo: 0xbb, hi: 0xbb}, + {value: 0x3000, lo: 0xbd, hi: 0xbf}, + // Block 0x91, offset 0x92 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x83}, + {value: 0x3000, lo: 0x85, hi: 0xbf}, + // Block 0x92, offset 0x93 + {value: 0x0000, lo: 0x06}, + {value: 0x3000, lo: 0x80, hi: 0x85}, + {value: 0x3000, lo: 0x87, hi: 0x8a}, + {value: 0x3000, lo: 0x8d, hi: 0x94}, + {value: 0x3000, lo: 0x96, hi: 0x9c}, + {value: 0x3000, lo: 0x9e, hi: 0xb9}, + {value: 0x3000, lo: 0xbb, hi: 0xbe}, + // Block 0x93, offset 0x94 + {value: 0x0000, lo: 0x04}, + {value: 0x3000, lo: 0x80, hi: 0x84}, + {value: 0x3000, lo: 0x86, hi: 0x86}, + {value: 0x3000, lo: 0x8a, hi: 0x90}, + {value: 0x3000, lo: 0x92, hi: 0xbf}, + // Block 0x94, offset 0x95 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0xa5}, + {value: 0x3000, lo: 0xa8, hi: 0xbf}, + // Block 0x95, offset 0x96 + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x8b}, + {value: 0x3000, lo: 0x8e, hi: 0xbf}, + // Block 0x96, offset 0x97 + {value: 0x0000, lo: 0x03}, + {value: 0x3000, lo: 0x80, hi: 0x8a}, + {value: 0x3000, lo: 0x90, hi: 0xae}, + {value: 0x3000, lo: 0xb0, hi: 0xbf}, + // Block 0x97, offset 0x98 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x80, hi: 0x8f}, + // Block 0x98, offset 0x99 + {value: 0x0000, lo: 0x01}, + {value: 0x3000, lo: 0x90, hi: 0x90}, + // Block 0x99, offset 0x9a + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x82}, + {value: 0x3000, lo: 0x90, hi: 0xba}, + // Block 0x9a, offset 0x9b + {value: 0x0000, lo: 0x02}, + {value: 0x3000, lo: 0x80, hi: 0x88}, + {value: 0x3000, lo: 0x90, hi: 0x91}, + // Block 0x9b, offset 0x9c + {value: 0x0000, lo: 0x01}, + {value: 0x3300, lo: 0x80, hi: 0x9d}, } // charInfoLookup: 1152 bytes @@ -6498,83 +6689,83 @@ var charInfoLookup = [1152]uint8{ // Block 0x1, offset 0x40 // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0x0c2: 0x03, 0x0c3: 0x04, 0x0c4: 0x05, 0x0c5: 0x06, 0x0c6: 0x07, 0x0c7: 0x08, - 0x0c8: 0x09, 0x0ca: 0x0a, 0x0cb: 0x0b, 0x0cc: 0x0c, 0x0cd: 0x0d, 0x0ce: 0x0e, 0x0cf: 0x0f, - 0x0d0: 0x10, 0x0d1: 0x11, 0x0d2: 0x12, 0x0d3: 0x13, 0x0d6: 0x14, 0x0d7: 0x15, - 0x0d8: 0x16, 0x0d9: 0x17, 0x0db: 0x18, 0x0dc: 0x19, 0x0dd: 0x1a, 0x0df: 0x1b, + 0x0c2: 0x10, 0x0c3: 0x03, 0x0c4: 0x11, 0x0c5: 0x12, 0x0c6: 0x13, 0x0c7: 0x14, + 0x0c8: 0x15, 0x0ca: 0x16, 0x0cb: 0x17, 0x0cc: 0x04, 0x0cd: 0x05, 0x0ce: 0x06, 0x0cf: 0x18, + 0x0d0: 0x07, 0x0d1: 0x19, 0x0d2: 0x1a, 0x0d3: 0x1b, 0x0d6: 0x08, 0x0d7: 0x1c, + 0x0d8: 0x1d, 0x0d9: 0x09, 0x0db: 0x1e, 0x0dc: 0x1f, 0x0dd: 0x20, 0x0df: 0x21, 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, 0x0ea: 0x08, 0x0eb: 0x09, 0x0ec: 0x09, 0x0ed: 0x0a, 0x0ef: 0x0b, 0x0f0: 0x11, // Block 0x4, offset 0x100 - 0x120: 0x1c, 0x121: 0x1d, 0x124: 0x1e, 0x125: 0x1f, 0x126: 0x20, 0x127: 0x21, - 0x128: 0x22, 0x129: 0x23, 0x12a: 0x24, 0x12b: 0x25, 0x12c: 0x20, 0x12d: 0x26, 0x12e: 0x27, 0x12f: 0x28, - 0x131: 0x29, 0x132: 0x2a, 0x133: 0x2b, 0x134: 0x2c, 0x135: 0x28, 0x137: 0x2d, - 0x138: 0x2e, 0x139: 0x2f, 0x13a: 0x30, 0x13b: 0x31, 0x13c: 0x32, 0x13d: 0x33, 0x13e: 0x34, 0x13f: 0x35, + 0x120: 0x22, 0x121: 0x23, 0x124: 0x24, 0x125: 0x25, 0x126: 0x26, 0x127: 0x27, + 0x128: 0x28, 0x129: 0x29, 0x12a: 0x2a, 0x12b: 0x2b, 0x12c: 0x26, 0x12d: 0x2c, 0x12e: 0x2d, 0x12f: 0x2e, + 0x131: 0x2f, 0x132: 0x30, 0x133: 0x31, 0x134: 0x32, 0x135: 0x2e, 0x137: 0x33, + 0x138: 0x34, 0x139: 0x35, 0x13a: 0x36, 0x13b: 0x37, 0x13c: 0x38, 0x13d: 0x39, 0x13e: 0x3a, 0x13f: 0x3b, // Block 0x5, offset 0x140 - 0x140: 0x36, 0x142: 0x37, 0x143: 0x38, 0x145: 0x39, 0x146: 0x3a, 0x147: 0x3b, - 0x14d: 0x3c, - 0x15c: 0x3d, 0x15f: 0x3e, - 0x162: 0x3f, 0x164: 0x40, - 0x168: 0x41, 0x169: 0x42, 0x16c: 0x43, 0x16d: 0x44, 0x16e: 0x45, 0x16f: 0x46, - 0x170: 0x47, 0x173: 0x48, 0x174: 0x49, 0x175: 0x4a, 0x176: 0x4b, 0x177: 0x4c, - 0x178: 0x4d, 0x179: 0x4e, 0x17a: 0x4f, 0x17b: 0x50, 0x17c: 0x51, 0x17d: 0x52, 0x17e: 0x53, 0x17f: 0x54, + 0x140: 0x3c, 0x142: 0x3d, 0x143: 0x3e, 0x144: 0x3f, 0x145: 0x40, 0x146: 0x41, 0x147: 0x42, + 0x14d: 0x43, + 0x15c: 0x44, 0x15f: 0x45, + 0x162: 0x46, 0x164: 0x47, + 0x168: 0x48, 0x169: 0x49, 0x16c: 0x4a, 0x16d: 0x4b, 0x16e: 0x4c, 0x16f: 0x4d, + 0x170: 0x4e, 0x173: 0x4f, 0x174: 0x50, 0x175: 0x51, 0x176: 0x52, 0x177: 0x53, + 0x178: 0x54, 0x179: 0x55, 0x17a: 0x56, 0x17b: 0x57, 0x17c: 0x58, 0x17d: 0x0a, 0x17e: 0x59, 0x17f: 0x0b, // Block 0x6, offset 0x180 - 0x180: 0x55, 0x181: 0x56, 0x182: 0x57, 0x183: 0x58, 0x184: 0x59, 0x185: 0x5a, 0x186: 0x5b, 0x187: 0x5c, - 0x188: 0x5d, 0x189: 0x5e, 0x18a: 0x5f, 0x18b: 0x60, 0x18c: 0x61, - 0x191: 0x62, 0x192: 0x63, 0x193: 0x64, - 0x1a8: 0x65, 0x1a9: 0x66, 0x1ab: 0x67, - 0x1b1: 0x68, 0x1b3: 0x69, 0x1b5: 0x6a, 0x1b7: 0x6b, - 0x1ba: 0x6c, 0x1bb: 0x6d, 0x1bc: 0x63, 0x1bd: 0x63, 0x1be: 0x63, 0x1bf: 0x6e, + 0x180: 0x5a, 0x181: 0x5b, 0x182: 0x5c, 0x183: 0x5d, 0x184: 0x5e, 0x185: 0x5f, 0x186: 0x60, 0x187: 0x61, + 0x188: 0x62, 0x189: 0x0c, 0x18a: 0x63, 0x18b: 0x64, 0x18c: 0x65, + 0x191: 0x66, 0x192: 0x67, 0x193: 0x68, + 0x1a8: 0x69, 0x1a9: 0x6a, 0x1ab: 0x6b, + 0x1b1: 0x6c, 0x1b3: 0x6d, 0x1b5: 0x6e, 0x1b7: 0x6f, + 0x1ba: 0x70, 0x1bb: 0x71, 0x1bc: 0x67, 0x1bd: 0x67, 0x1be: 0x67, 0x1bf: 0x72, // Block 0x7, offset 0x1c0 - 0x1c0: 0x6f, 0x1c1: 0x70, 0x1c2: 0x71, 0x1c3: 0x72, 0x1c4: 0x73, 0x1c5: 0x63, 0x1c6: 0x74, - 0x1c8: 0x75, 0x1c9: 0x76, 0x1ca: 0x63, 0x1cb: 0x77, 0x1cc: 0x63, 0x1cd: 0x63, 0x1ce: 0x63, 0x1cf: 0x63, + 0x1c0: 0x73, 0x1c1: 0x0d, 0x1c2: 0x0e, 0x1c3: 0x0f, 0x1c4: 0x74, 0x1c5: 0x67, 0x1c6: 0x75, + 0x1c8: 0x76, 0x1c9: 0x77, 0x1ca: 0x67, 0x1cb: 0x78, 0x1cc: 0x67, 0x1cd: 0x67, 0x1ce: 0x67, 0x1cf: 0x67, // Block 0x8, offset 0x200 - 0x219: 0x78, 0x21b: 0x79, 0x21d: 0x7a, - 0x220: 0x7b, 0x223: 0x7c, 0x224: 0x7d, 0x225: 0x7e, 0x226: 0x7f, 0x227: 0x80, - 0x22a: 0x81, 0x22b: 0x82, 0x22f: 0x83, - 0x230: 0x84, 0x231: 0x84, 0x232: 0x84, 0x233: 0x84, 0x234: 0x84, 0x235: 0x84, 0x236: 0x84, 0x237: 0x84, - 0x238: 0x84, 0x239: 0x84, 0x23a: 0x84, 0x23b: 0x84, 0x23c: 0x84, 0x23d: 0x84, 0x23e: 0x84, 0x23f: 0x84, + 0x219: 0x79, 0x21b: 0x7a, 0x21d: 0x7b, + 0x220: 0x7c, 0x223: 0x7d, 0x224: 0x7e, 0x225: 0x7f, 0x226: 0x80, 0x227: 0x81, + 0x22a: 0x82, 0x22b: 0x83, 0x22f: 0x84, + 0x230: 0x85, 0x231: 0x85, 0x232: 0x85, 0x233: 0x85, 0x234: 0x85, 0x235: 0x85, 0x236: 0x85, 0x237: 0x85, + 0x238: 0x85, 0x239: 0x85, 0x23a: 0x85, 0x23b: 0x85, 0x23c: 0x85, 0x23d: 0x85, 0x23e: 0x85, 0x23f: 0x85, // Block 0x9, offset 0x240 - 0x240: 0x84, 0x241: 0x84, 0x242: 0x84, 0x243: 0x84, 0x244: 0x84, 0x245: 0x84, 0x246: 0x84, 0x247: 0x84, - 0x248: 0x84, 0x249: 0x84, 0x24a: 0x84, 0x24b: 0x84, 0x24c: 0x84, 0x24d: 0x84, 0x24e: 0x84, 0x24f: 0x84, - 0x250: 0x84, 0x251: 0x84, 0x252: 0x84, 0x253: 0x84, 0x254: 0x84, 0x255: 0x84, 0x256: 0x84, 0x257: 0x84, - 0x258: 0x84, 0x259: 0x84, 0x25a: 0x84, 0x25b: 0x84, 0x25c: 0x84, 0x25d: 0x84, 0x25e: 0x84, 0x25f: 0x84, - 0x260: 0x84, 0x261: 0x84, 0x262: 0x84, 0x263: 0x84, 0x264: 0x84, 0x265: 0x84, 0x266: 0x84, 0x267: 0x84, - 0x268: 0x84, 0x269: 0x84, 0x26a: 0x84, 0x26b: 0x84, 0x26c: 0x84, 0x26d: 0x84, 0x26e: 0x84, 0x26f: 0x84, - 0x270: 0x84, 0x271: 0x84, 0x272: 0x84, 0x273: 0x84, 0x274: 0x84, 0x275: 0x84, 0x276: 0x84, 0x277: 0x84, - 0x278: 0x84, 0x279: 0x84, 0x27a: 0x84, 0x27b: 0x84, 0x27c: 0x84, 0x27d: 0x84, 0x27e: 0x84, 0x27f: 0x84, + 0x240: 0x85, 0x241: 0x85, 0x242: 0x85, 0x243: 0x85, 0x244: 0x85, 0x245: 0x85, 0x246: 0x85, 0x247: 0x85, + 0x248: 0x85, 0x249: 0x85, 0x24a: 0x85, 0x24b: 0x85, 0x24c: 0x85, 0x24d: 0x85, 0x24e: 0x85, 0x24f: 0x85, + 0x250: 0x85, 0x251: 0x85, 0x252: 0x85, 0x253: 0x85, 0x254: 0x85, 0x255: 0x85, 0x256: 0x85, 0x257: 0x85, + 0x258: 0x85, 0x259: 0x85, 0x25a: 0x85, 0x25b: 0x85, 0x25c: 0x85, 0x25d: 0x85, 0x25e: 0x85, 0x25f: 0x85, + 0x260: 0x85, 0x261: 0x85, 0x262: 0x85, 0x263: 0x85, 0x264: 0x85, 0x265: 0x85, 0x266: 0x85, 0x267: 0x85, + 0x268: 0x85, 0x269: 0x85, 0x26a: 0x85, 0x26b: 0x85, 0x26c: 0x85, 0x26d: 0x85, 0x26e: 0x85, 0x26f: 0x85, + 0x270: 0x85, 0x271: 0x85, 0x272: 0x85, 0x273: 0x85, 0x274: 0x85, 0x275: 0x85, 0x276: 0x85, 0x277: 0x85, + 0x278: 0x85, 0x279: 0x85, 0x27a: 0x85, 0x27b: 0x85, 0x27c: 0x85, 0x27d: 0x85, 0x27e: 0x85, 0x27f: 0x85, // Block 0xa, offset 0x280 - 0x280: 0x84, 0x281: 0x84, 0x282: 0x84, 0x283: 0x84, 0x284: 0x84, 0x285: 0x84, 0x286: 0x84, 0x287: 0x84, - 0x288: 0x84, 0x289: 0x84, 0x28a: 0x84, 0x28b: 0x84, 0x28c: 0x84, 0x28d: 0x84, 0x28e: 0x84, 0x28f: 0x84, - 0x290: 0x84, 0x291: 0x84, 0x292: 0x84, 0x293: 0x84, 0x294: 0x84, 0x295: 0x84, 0x296: 0x84, 0x297: 0x84, - 0x298: 0x84, 0x299: 0x84, 0x29a: 0x84, 0x29b: 0x84, 0x29c: 0x84, 0x29d: 0x84, 0x29e: 0x85, + 0x280: 0x85, 0x281: 0x85, 0x282: 0x85, 0x283: 0x85, 0x284: 0x85, 0x285: 0x85, 0x286: 0x85, 0x287: 0x85, + 0x288: 0x85, 0x289: 0x85, 0x28a: 0x85, 0x28b: 0x85, 0x28c: 0x85, 0x28d: 0x85, 0x28e: 0x85, 0x28f: 0x85, + 0x290: 0x85, 0x291: 0x85, 0x292: 0x85, 0x293: 0x85, 0x294: 0x85, 0x295: 0x85, 0x296: 0x85, 0x297: 0x85, + 0x298: 0x85, 0x299: 0x85, 0x29a: 0x85, 0x29b: 0x85, 0x29c: 0x85, 0x29d: 0x85, 0x29e: 0x86, // Block 0xb, offset 0x2c0 - 0x2e4: 0x86, 0x2e5: 0x86, 0x2e6: 0x86, 0x2e7: 0x86, - 0x2e8: 0x87, 0x2e9: 0x88, 0x2ea: 0x86, 0x2eb: 0x89, 0x2ec: 0x8a, 0x2ed: 0x8b, 0x2ee: 0x8c, 0x2ef: 0x8d, - 0x2f0: 0x63, 0x2f1: 0x63, 0x2f2: 0x63, 0x2f3: 0x63, 0x2f4: 0x8e, 0x2f5: 0x8f, 0x2f6: 0x90, 0x2f7: 0x91, - 0x2f8: 0x92, 0x2f9: 0x93, 0x2fa: 0x63, 0x2fb: 0x94, 0x2fc: 0x95, 0x2fd: 0x63, 0x2fe: 0x77, 0x2ff: 0x96, + 0x2e4: 0x87, 0x2e5: 0x87, 0x2e6: 0x87, 0x2e7: 0x87, + 0x2e8: 0x88, 0x2e9: 0x89, 0x2ea: 0x87, 0x2eb: 0x8a, 0x2ec: 0x8b, 0x2ed: 0x8c, 0x2ee: 0x8d, 0x2ef: 0x8e, + 0x2f0: 0x67, 0x2f1: 0x67, 0x2f2: 0x67, 0x2f3: 0x67, 0x2f4: 0x8f, 0x2f5: 0x90, 0x2f6: 0x91, 0x2f7: 0x92, + 0x2f8: 0x93, 0x2f9: 0x94, 0x2fa: 0x67, 0x2fb: 0x95, 0x2fc: 0x96, 0x2fd: 0x67, 0x2fe: 0x78, 0x2ff: 0x97, // Block 0xc, offset 0x300 - 0x307: 0x97, - 0x328: 0x98, + 0x307: 0x98, + 0x328: 0x99, // Block 0xd, offset 0x340 - 0x341: 0x7b, 0x342: 0x99, + 0x341: 0x7c, 0x342: 0x9a, // Block 0xe, offset 0x380 - 0x385: 0x9a, 0x386: 0x9b, 0x387: 0x9c, - 0x389: 0x9d, - 0x390: 0x63, 0x391: 0x9e, 0x392: 0x9f, 0x393: 0xa0, 0x394: 0xa1, 0x395: 0xa2, 0x396: 0x63, 0x397: 0x63, - 0x398: 0x63, 0x399: 0x63, 0x39a: 0xa3, 0x39b: 0x63, 0x39c: 0x63, 0x39d: 0x63, 0x39e: 0x63, 0x39f: 0xa4, + 0x385: 0x9b, 0x386: 0x9c, 0x387: 0x9d, + 0x389: 0x9e, + 0x390: 0x67, 0x391: 0x9f, 0x392: 0xa0, 0x393: 0xa1, 0x394: 0xa2, 0x395: 0xa3, 0x396: 0x67, 0x397: 0x67, + 0x398: 0x67, 0x399: 0x67, 0x39a: 0xa4, 0x39b: 0x67, 0x39c: 0x67, 0x39d: 0x67, 0x39e: 0x67, 0x39f: 0xa5, // Block 0xf, offset 0x3c0 - 0x3c4: 0xa5, 0x3c5: 0xa6, 0x3c6: 0xa7, - 0x3c8: 0xa8, 0x3c9: 0xa9, + 0x3c4: 0xa6, 0x3c5: 0xa7, 0x3c6: 0xa8, + 0x3c8: 0xa9, 0x3c9: 0xaa, // Block 0x10, offset 0x400 - 0x420: 0x86, 0x421: 0x86, 0x422: 0x86, 0x423: 0x86, 0x424: 0x86, 0x425: 0x86, 0x426: 0x86, 0x427: 0x86, - 0x428: 0xaa, + 0x420: 0x87, 0x421: 0x87, 0x422: 0x87, 0x423: 0x87, 0x424: 0x87, 0x425: 0x87, 0x426: 0x87, 0x427: 0x87, + 0x428: 0xab, // Block 0x11, offset 0x440 0x450: 0x0c, 0x451: 0x0d, 0x45d: 0x0e, 0x45f: 0x0f, 0x46f: 0x10, } -var charInfoTrie = trie{charInfoLookup[:], charInfoValues[:]} +var charInfoTrie = trie{charInfoLookup[:], charInfoValues[:], charInfoSparseValues[:], charInfoSparseOffset[:], 16} -// Total size of tables: 78KB (80234 bytes) +// Total size of tables: 48KB (48756 bytes) diff --git a/libgo/go/exp/norm/trie.go b/libgo/go/exp/norm/trie.go index 6b65401..93cb9c3 100644 --- a/libgo/go/exp/norm/trie.go +++ b/libgo/go/exp/norm/trie.go @@ -4,9 +4,44 @@ package norm +type valueRange struct { + value uint16 // header: value:stride + lo, hi byte // header: lo:n +} + type trie struct { - index []uint8 - values []uint16 + index []uint8 + values []uint16 + sparse []valueRange + sparseOffset []uint16 + cutoff uint8 // indices >= cutoff are sparse +} + +// lookupValue determines the type of block n and looks up the value for b. +// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block +// is a list of ranges with an accompanying value. Given a matching range r, +// the value for b is by r.value + (b - r.lo) * stride. +func (t *trie) lookupValue(n uint8, b byte) uint16 { + if n < t.cutoff { + return t.values[uint16(n)<<6+uint16(b&maskx)] + } + offset := t.sparseOffset[n-t.cutoff] + header := t.sparse[offset] + lo := offset + 1 + hi := lo + uint16(header.lo) + for lo < hi { + m := lo + (hi-lo)/2 + r := t.sparse[m] + if r.lo <= b && b <= r.hi { + return r.value + uint16(b-r.lo)*header.value + } + if b < r.lo { + hi = m + } else { + lo = m + 1 + } + } + return 0 } const ( @@ -44,8 +79,7 @@ func (t *trie) lookup(s []byte) (v uint16, sz int) { if c1 < tx || t2 <= c1 { return 0, 1 } - o := uint16(i)<<6 + uint16(c1)&maskx - return t.values[o], 2 + return t.lookupValue(i, c1), 2 case c0 < t4: if len(s) < 3 { return 0, 0 @@ -61,8 +95,7 @@ func (t *trie) lookup(s []byte) (v uint16, sz int) { if c2 < tx || t2 <= c2 { return 0, 2 } - o = uint16(i)<<6 + uint16(c2)&maskx - return t.values[o], 3 + return t.lookupValue(i, c2), 3 case c0 < t5: if len(s) < 4 { return 0, 0 @@ -84,18 +117,7 @@ func (t *trie) lookup(s []byte) (v uint16, sz int) { if c3 < tx || t2 <= c3 { return 0, 3 } - o = uint16(i)<<6 + uint16(c3)&maskx - return t.values[o], 4 - case c0 < t6: - if len(s) < 5 { - return 0, 0 - } - return 0, 5 - case c0 < te: - if len(s) < 6 { - return 0, 0 - } - return 0, 6 + return t.lookupValue(i, c3), 4 } // Illegal rune return 0, 1 @@ -120,8 +142,7 @@ func (t *trie) lookupString(s string) (v uint16, sz int) { if c1 < tx || t2 <= c1 { return 0, 1 } - o := uint16(i)<<6 + uint16(c1)&maskx - return t.values[o], 2 + return t.lookupValue(i, c1), 2 case c0 < t4: if len(s) < 3 { return 0, 0 @@ -137,8 +158,7 @@ func (t *trie) lookupString(s string) (v uint16, sz int) { if c2 < tx || t2 <= c2 { return 0, 2 } - o = uint16(i)<<6 + uint16(c2)&maskx - return t.values[o], 3 + return t.lookupValue(i, c2), 3 case c0 < t5: if len(s) < 4 { return 0, 0 @@ -160,18 +180,7 @@ func (t *trie) lookupString(s string) (v uint16, sz int) { if c3 < tx || t2 <= c3 { return 0, 3 } - o = uint16(i)<<6 + uint16(c3)&maskx - return t.values[o], 4 - case c0 < t6: - if len(s) < 5 { - return 0, 0 - } - return 0, 5 - case c0 < te: - if len(s) < 6 { - return 0, 0 - } - return 0, 6 + return t.lookupValue(i, c3), 4 } // Illegal rune return 0, 1 @@ -188,19 +197,16 @@ func (t *trie) lookupUnsafe(s []byte) uint16 { return 0 } i := t.index[c0] - o := uint16(i)<<6 + uint16(s[1])&maskx if c0 < t3 { - return t.values[o] + return t.lookupValue(i, s[1]) } - i = t.index[o] - o = uint16(i)<<6 + uint16(s[2])&maskx + i = t.index[uint16(i)<<6+uint16(s[1])&maskx] if c0 < t4 { - return t.values[o] + return t.lookupValue(i, s[2]) } - i = t.index[o] - o = uint16(i)<<6 + uint16(s[3])&maskx + i = t.index[uint16(i)<<6+uint16(s[2])&maskx] if c0 < t5 { - return t.values[o] + return t.lookupValue(i, s[3]) } return 0 } @@ -216,19 +222,16 @@ func (t *trie) lookupStringUnsafe(s string) uint16 { return 0 } i := t.index[c0] - o := uint16(i)<<6 + uint16(s[1])&maskx if c0 < t3 { - return t.values[o] + return t.lookupValue(i, s[1]) } - i = t.index[o] - o = uint16(i)<<6 + uint16(s[2])&maskx + i = t.index[uint16(i)<<6+uint16(s[1])&maskx] if c0 < t4 { - return t.values[o] + return t.lookupValue(i, s[2]) } - i = t.index[o] - o = uint16(i)<<6 + uint16(s[3])&maskx + i = t.index[uint16(i)<<6+uint16(s[2])&maskx] if c0 < t5 { - return t.values[o] + return t.lookupValue(i, s[3]) } return 0 } diff --git a/libgo/go/exp/norm/trie_test.go b/libgo/go/exp/norm/trie_test.go index ad87d97..5649fb7 100644 --- a/libgo/go/exp/norm/trie_test.go +++ b/libgo/go/exp/norm/trie_test.go @@ -8,6 +8,41 @@ import ( // Test data is located in triedata_test.go; generated by maketesttables. var testdata = testdataTrie +type rangeTest struct { + block uint8 + lookup byte + result uint16 + table []valueRange + offsets []uint16 +} + +var range1Off = []uint16{0, 2} +var range1 = []valueRange{ + {0, 1, 0}, + {1, 0x80, 0x80}, + {0, 2, 0}, + {1, 0x80, 0x80}, + {9, 0xff, 0xff}, +} + +var rangeTests = []rangeTest{ + {10, 0x80, 1, range1, range1Off}, + {10, 0x00, 0, range1, range1Off}, + {11, 0x80, 1, range1, range1Off}, + {11, 0xff, 9, range1, range1Off}, + {11, 0x00, 0, range1, range1Off}, +} + +func TestLookupSparse(t *testing.T) { + for i, test := range rangeTests { + n := trie{sparse: test.table, sparseOffset: test.offsets, cutoff: 10} + v := n.lookupValue(test.block, test.lookup) + if v != test.result { + t.Errorf("LookupSparse:%d: found %X; want %X", i, v, test.result) + } + } +} + // Test cases for illegal runes. type trietest struct { size int @@ -32,8 +67,10 @@ var tests = []trietest{ {0, []byte{t2}}, {0, []byte{t3, tx}}, {0, []byte{t4, tx, tx}}, - {0, []byte{t5, tx, tx, tx}}, - {0, []byte{t6, tx, tx, tx, tx}}, + + // we only support UTF-8 up to utf8.UTFMax bytes (4 bytes) + {1, []byte{t5, tx, tx, tx, tx}}, + {1, []byte{t6, tx, tx, tx, tx, tx}}, } func mkUtf8(rune int) ([]byte, int) { @@ -47,10 +84,10 @@ func TestLookup(t *testing.T) { b, szg := mkUtf8(tt) v, szt := testdata.lookup(b) if int(v) != i { - t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i) + t.Errorf("lookup(%U): found value %#x, expected %#x", tt, v, i) } if szt != szg { - t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg) + t.Errorf("lookup(%U): found size %d, expected %d", tt, szt, szg) } } for i, tt := range tests { diff --git a/libgo/go/exp/norm/triedata_test.go b/libgo/go/exp/norm/triedata_test.go index f886e60..e8898e5 100644 --- a/libgo/go/exp/norm/triedata_test.go +++ b/libgo/go/exp/norm/triedata_test.go @@ -4,34 +4,55 @@ package norm -var testRunes = []int{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111} +var testRunes = []int{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111, 512, 513, 514, 528, 533} -// testdataValues: 768 entries, 1536 bytes +// testdataValues: 192 entries, 384 bytes // Block 2 is the null block. -var testdataValues = [768]uint16{ +var testdataValues = [192]uint16{ // Block 0x0, offset 0x0 0x000c: 0x0001, // Block 0x1, offset 0x40 0x007f: 0x0002, // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0x00c0: 0x0003, - // Block 0x4, offset 0x100 - 0x0100: 0x0004, - // Block 0x5, offset 0x140 - 0x017f: 0x0005, - // Block 0x6, offset 0x180 - 0x0180: 0x0006, - // Block 0x7, offset 0x1c0 - 0x01d9: 0x0007, - // Block 0x8, offset 0x200 - 0x023f: 0x0008, - // Block 0x9, offset 0x240 - 0x0240: 0x0009, - // Block 0xa, offset 0x280 - 0x0281: 0x000a, - // Block 0xb, offset 0x2c0 - 0x02ff: 0x000b, +} + +// testdataSparseOffset: 10 entries, 20 bytes +var testdataSparseOffset = []uint16{0x0, 0x2, 0x4, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14} + +// testdataSparseValues: 22 entries, 88 bytes +var testdataSparseValues = [22]valueRange{ + // Block 0x0, offset 0x1 + {value: 0x0000, lo: 0x01}, + {value: 0x0003, lo: 0x80, hi: 0x80}, + // Block 0x1, offset 0x2 + {value: 0x0000, lo: 0x01}, + {value: 0x0004, lo: 0x80, hi: 0x80}, + // Block 0x2, offset 0x3 + {value: 0x0001, lo: 0x03}, + {value: 0x000c, lo: 0x80, hi: 0x82}, + {value: 0x000f, lo: 0x90, hi: 0x90}, + {value: 0x0010, lo: 0x95, hi: 0x95}, + // Block 0x3, offset 0x4 + {value: 0x0000, lo: 0x01}, + {value: 0x0005, lo: 0xbf, hi: 0xbf}, + // Block 0x4, offset 0x5 + {value: 0x0000, lo: 0x01}, + {value: 0x0006, lo: 0x80, hi: 0x80}, + // Block 0x5, offset 0x6 + {value: 0x0000, lo: 0x01}, + {value: 0x0007, lo: 0x99, hi: 0x99}, + // Block 0x6, offset 0x7 + {value: 0x0000, lo: 0x01}, + {value: 0x0008, lo: 0xbf, hi: 0xbf}, + // Block 0x7, offset 0x8 + {value: 0x0000, lo: 0x01}, + {value: 0x0009, lo: 0x80, hi: 0x80}, + // Block 0x8, offset 0x9 + {value: 0x0000, lo: 0x01}, + {value: 0x000a, lo: 0x81, hi: 0x81}, + // Block 0x9, offset 0xa + {value: 0x0000, lo: 0x01}, + {value: 0x000b, lo: 0xbf, hi: 0xbf}, } // testdataLookup: 640 bytes @@ -42,22 +63,23 @@ var testdataLookup = [640]uint8{ // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 0x0c2: 0x03, 0x0c4: 0x04, - 0x0df: 0x05, + 0x0c8: 0x05, + 0x0df: 0x06, 0x0e0: 0x04, 0x0ef: 0x05, 0x0f0: 0x07, 0x0f4: 0x09, // Block 0x4, offset 0x100 - 0x120: 0x06, 0x126: 0x07, + 0x120: 0x07, 0x126: 0x08, // Block 0x5, offset 0x140 - 0x17f: 0x08, + 0x17f: 0x09, // Block 0x6, offset 0x180 - 0x180: 0x09, 0x184: 0x0a, + 0x180: 0x0a, 0x184: 0x0b, // Block 0x7, offset 0x1c0 0x1d0: 0x06, // Block 0x8, offset 0x200 - 0x23f: 0x0b, + 0x23f: 0x0c, // Block 0x9, offset 0x240 0x24f: 0x08, } -var testdataTrie = trie{testdataLookup[:], testdataValues[:]} +var testdataTrie = trie{testdataLookup[:], testdataValues[:], testdataSparseValues[:], testdataSparseOffset[:], 3} diff --git a/libgo/go/exp/norm/triegen.go b/libgo/go/exp/norm/triegen.go index 2b7eeee..515e1c7 100644 --- a/libgo/go/exp/norm/triegen.go +++ b/libgo/go/exp/norm/triegen.go @@ -17,10 +17,13 @@ import ( "utf8" ) +const blockSize = 64 +const maxSparseEntries = 16 + // Intermediate trie structure type trieNode struct { table [256]*trieNode - value uint16 + value int b byte leaf bool } @@ -53,6 +56,44 @@ func (n trieNode) isInternal() bool { return internal } +func (n trieNode) mostFrequentStride() int { + counts := make(map[int]int) + v := 0 + for _, t := range n.table[0x80 : 0x80+blockSize] { + if t != nil { + if stride := t.value - v; v != 0 && stride >= 0 { + counts[stride]++ + } + v = t.value + } + } + var maxs, maxc int + for stride, cnt := range counts { + if cnt > maxc { + maxs, maxc = stride, cnt + } + } + return maxs +} + +func (n trieNode) countSparseEntries() int { + stride := n.mostFrequentStride() + var count, v int + for _, t := range n.table[0x80 : 0x80+blockSize] { + tv := 0 + if t != nil { + tv = t.value + } + if tv-v != stride { + if tv != 0 { + count++ + } + } + v = tv + } + return count +} + func (n *trieNode) insert(rune int, value uint16) { var p [utf8.UTFMax]byte sz := utf8.EncodeRune(p[:], rune) @@ -69,35 +110,40 @@ func (n *trieNode) insert(rune int, value uint16) { } n = nn } - n.value = value + n.value = int(value) n.leaf = true } type nodeIndex struct { lookupBlocks []*trieNode valueBlocks []*trieNode + sparseBlocks []*trieNode + sparseOffset []uint16 + sparseCount int - lookupBlockIdx map[uint32]uint16 - valueBlockIdx map[uint32]uint16 + lookupBlockIdx map[uint32]int + valueBlockIdx map[uint32]int } func newIndex() *nodeIndex { index := &nodeIndex{} index.lookupBlocks = make([]*trieNode, 0) index.valueBlocks = make([]*trieNode, 0) - index.lookupBlockIdx = make(map[uint32]uint16) - index.valueBlockIdx = make(map[uint32]uint16) + index.sparseBlocks = make([]*trieNode, 0) + index.sparseOffset = make([]uint16, 1) + index.lookupBlockIdx = make(map[uint32]int) + index.valueBlockIdx = make(map[uint32]int) return index } -func computeOffsets(index *nodeIndex, n *trieNode) uint16 { +func computeOffsets(index *nodeIndex, n *trieNode) int { if n.leaf { return n.value } hasher := crc32.New(crc32.MakeTable(crc32.IEEE)) // We only index continuation bytes. - for i := 0; i < 64; i++ { - var v uint16 = 0 + for i := 0; i < blockSize; i++ { + v := 0 if nn := n.table[0x80+i]; nn != nil { v = computeOffsets(index, nn) } @@ -107,7 +153,7 @@ func computeOffsets(index *nodeIndex, n *trieNode) uint16 { if n.isInternal() { v, ok := index.lookupBlockIdx[h] if !ok { - v = uint16(len(index.lookupBlocks)) + v = len(index.lookupBlocks) index.lookupBlocks = append(index.lookupBlocks, n) index.lookupBlockIdx[h] = v } @@ -115,9 +161,17 @@ func computeOffsets(index *nodeIndex, n *trieNode) uint16 { } else { v, ok := index.valueBlockIdx[h] if !ok { - v = uint16(len(index.valueBlocks)) - index.valueBlocks = append(index.valueBlocks, n) - index.valueBlockIdx[h] = v + if c := n.countSparseEntries(); c > maxSparseEntries { + v = len(index.valueBlocks) + index.valueBlocks = append(index.valueBlocks, n) + index.valueBlockIdx[h] = v + } else { + v = -len(index.sparseOffset) + index.sparseBlocks = append(index.sparseBlocks, n) + index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount)) + index.sparseCount += c + 1 + index.valueBlockIdx[h] = v + } } n.value = v } @@ -125,14 +179,14 @@ func computeOffsets(index *nodeIndex, n *trieNode) uint16 { } func printValueBlock(nr int, n *trieNode, offset int) { - boff := nr * 64 + boff := nr * blockSize fmt.Printf("\n// Block %#x, offset %#x", nr, boff) var printnewline bool - for i := 0; i < 64; i++ { + for i := 0; i < blockSize; i++ { if i%6 == 0 { printnewline = true } - v := uint16(0) + v := 0 if nn := n.table[i+offset]; nn != nil { v = nn.value } @@ -141,24 +195,55 @@ func printValueBlock(nr int, n *trieNode, offset int) { fmt.Printf("\n") printnewline = false } - fmt.Printf("%#04x:%#04x, ", nr*64+i, v) + fmt.Printf("%#04x:%#04x, ", boff+i, v) } } } -func printLookupBlock(nr int, n *trieNode, offset int) { - boff := nr * 64 +func printSparseBlock(nr int, n *trieNode) { + boff := -n.value + fmt.Printf("\n// Block %#x, offset %#x", nr, boff) + v := 0 + //stride := f(n) + stride := n.mostFrequentStride() + c := n.countSparseEntries() + fmt.Printf("\n{value:%#04x,lo:%#02x},", stride, uint8(c)) + for i, nn := range n.table[0x80 : 0x80+blockSize] { + nv := 0 + if nn != nil { + nv = nn.value + } + if nv-v != stride { + if v != 0 { + fmt.Printf(",hi:%#02x},", 0x80+i-1) + } + if nv != 0 { + fmt.Printf("\n{value:%#04x,lo:%#02x", nv, nn.b) + } + } + v = nv + } + if v != 0 { + fmt.Printf(",hi:%#02x},", 0x80+blockSize-1) + } +} + +func printLookupBlock(nr int, n *trieNode, offset, cutoff int) { + boff := nr * blockSize fmt.Printf("\n// Block %#x, offset %#x", nr, boff) var printnewline bool - for i := 0; i < 64; i++ { + for i := 0; i < blockSize; i++ { if i%8 == 0 { printnewline = true } - v := uint16(0) + v := 0 if nn := n.table[i+offset]; nn != nil { v = nn.value } if v != 0 { + if v < 0 { + v = -v - 1 + cutoff + } if printnewline { fmt.Printf("\n") printnewline = false @@ -182,7 +267,7 @@ func (t *trieNode) printTables(name string) int { } } - nv := len(index.valueBlocks) * 64 + nv := len(index.valueBlocks) * blockSize fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2) fmt.Printf("// Block 2 is the null block.\n") fmt.Printf("var %sValues = [%d]uint16 {", name, nv) @@ -194,18 +279,32 @@ func (t *trieNode) printTables(name string) int { } fmt.Print("\n}\n\n") - ni := len(index.lookupBlocks) * 64 + ls := len(index.sparseBlocks) + fmt.Printf("// %sSparseOffset: %d entries, %d bytes\n", name, ls, ls*2) + fmt.Printf("var %sSparseOffset = %#v\n\n", name, index.sparseOffset[1:]) + + ns := index.sparseCount + fmt.Printf("// %sSparseValues: %d entries, %d bytes\n", name, ns, ns*4) + fmt.Printf("var %sSparseValues = [%d]valueRange {", name, ns) + for i, n := range index.sparseBlocks { + printSparseBlock(i, n) + } + fmt.Print("\n}\n\n") + + cutoff := len(index.valueBlocks) + ni := len(index.lookupBlocks) * blockSize fmt.Printf("// %sLookup: %d bytes\n", name, ni) fmt.Printf("// Block 0 is the null block.\n") fmt.Printf("var %sLookup = [%d]uint8 {", name, ni) - printLookupBlock(0, newNode(), 0) - printLookupBlock(1, newNode(), 0) - printLookupBlock(2, newNode(), 0) - printLookupBlock(3, t, 0xC0) + printLookupBlock(0, newNode(), 0, cutoff) + printLookupBlock(1, newNode(), 0, cutoff) + printLookupBlock(2, newNode(), 0, cutoff) + printLookupBlock(3, t, 0xC0, cutoff) for i := 4; i < len(index.lookupBlocks); i++ { - printLookupBlock(i, index.lookupBlocks[i], 0x80) + printLookupBlock(i, index.lookupBlocks[i], 0x80, cutoff) } fmt.Print("\n}\n\n") - fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:] }\n\n", name, name, name) - return nv*2 + ni + fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:], %sSparseValues[:], %sSparseOffset[:], %d}\n\n", + name, name, name, name, name, cutoff) + return nv*2 + ns*4 + ni + ls*2 } diff --git a/libgo/go/http/spdy/read.go b/libgo/go/exp/spdy/read.go index c6b6ab3..2b1fd3d 100644 --- a/libgo/go/http/spdy/read.go +++ b/libgo/go/exp/spdy/read.go @@ -214,7 +214,7 @@ func (f *Framer) readSynStreamFrame(h ControlFrameHeader, frame *SynStreamFrame) } // Remove this condition when we bump Version to 3. if Version >= 3 { - for h, _ := range frame.Headers { + for h := range frame.Headers { if invalidReqHeaders[h] { return &Error{InvalidHeaderPresent, frame.StreamId} } @@ -247,7 +247,7 @@ func (f *Framer) readSynReplyFrame(h ControlFrameHeader, frame *SynReplyFrame) o } // Remove this condition when we bump Version to 3. if Version >= 3 { - for h, _ := range frame.Headers { + for h := range frame.Headers { if invalidRespHeaders[h] { return &Error{InvalidHeaderPresent, frame.StreamId} } @@ -287,7 +287,7 @@ func (f *Framer) readHeadersFrame(h ControlFrameHeader, frame *HeadersFrame) os. } else { invalidHeaders = invalidRespHeaders } - for h, _ := range frame.Headers { + for h := range frame.Headers { if invalidHeaders[h] { return &Error{InvalidHeaderPresent, frame.StreamId} } diff --git a/libgo/go/http/spdy/spdy_test.go b/libgo/go/exp/spdy/spdy_test.go index cb91e02..cb91e02 100644 --- a/libgo/go/http/spdy/spdy_test.go +++ b/libgo/go/exp/spdy/spdy_test.go diff --git a/libgo/go/http/spdy/types.go b/libgo/go/exp/spdy/types.go index 41cafb1..41cafb1 100644 --- a/libgo/go/http/spdy/types.go +++ b/libgo/go/exp/spdy/types.go diff --git a/libgo/go/http/spdy/write.go b/libgo/go/exp/spdy/write.go index 7d40bbe..7d40bbe 100644 --- a/libgo/go/http/spdy/write.go +++ b/libgo/go/exp/spdy/write.go diff --git a/libgo/go/exp/sql/convert.go b/libgo/go/exp/sql/convert.go new file mode 100644 index 0000000..a35e0be --- /dev/null +++ b/libgo/go/exp/sql/convert.go @@ -0,0 +1,106 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Type conversions for Scan. + +package sql + +import ( + "fmt" + "os" + "reflect" + "strconv" +) + +// convertAssign copies to dest the value in src, converting it if possible. +// An error is returned if the copy would result in loss of information. +// dest should be a pointer type. +func convertAssign(dest, src interface{}) os.Error { + // Common cases, without reflect. Fall through. + switch s := src.(type) { + case string: + switch d := dest.(type) { + case *string: + *d = s + return nil + } + case []byte: + switch d := dest.(type) { + case *string: + *d = string(s) + return nil + case *[]byte: + *d = s + return nil + } + } + + sv := reflect.ValueOf(src) + + switch d := dest.(type) { + case *string: + switch sv.Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + *d = fmt.Sprintf("%v", src) + return nil + } + } + + if scanner, ok := dest.(ScannerInto); ok { + return scanner.ScanInto(src) + } + + dpv := reflect.ValueOf(dest) + if dpv.Kind() != reflect.Ptr { + return os.NewError("destination not a pointer") + } + + dv := reflect.Indirect(dpv) + if dv.Kind() == sv.Kind() { + dv.Set(sv) + return nil + } + + switch dv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if s, ok := asString(src); ok { + i64, err := strconv.Atoi64(s) + if err != nil { + return fmt.Errorf("converting string %q to a %s: %v", s, dv.Kind(), err) + } + if dv.OverflowInt(i64) { + return fmt.Errorf("string %q overflows %s", s, dv.Kind()) + } + dv.SetInt(i64) + return nil + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if s, ok := asString(src); ok { + u64, err := strconv.Atoui64(s) + if err != nil { + return fmt.Errorf("converting string %q to a %s: %v", s, dv.Kind(), err) + } + if dv.OverflowUint(u64) { + return fmt.Errorf("string %q overflows %s", s, dv.Kind()) + } + dv.SetUint(u64) + return nil + } + } + + return fmt.Errorf("unsupported driver -> Scan pair: %T -> %T", src, dest) +} + +func asString(src interface{}) (s string, ok bool) { + switch v := src.(type) { + case string: + return v, true + case []byte: + return string(v), true + } + return "", false +} diff --git a/libgo/go/exp/sql/convert_test.go b/libgo/go/exp/sql/convert_test.go new file mode 100644 index 0000000..88ba8e7 --- /dev/null +++ b/libgo/go/exp/sql/convert_test.go @@ -0,0 +1,108 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sql + +import ( + "fmt" + "reflect" + "testing" +) + +type conversionTest struct { + s, d interface{} // source and destination + + // following are used if they're non-zero + wantint int64 + wantuint uint64 + wantstr string + wanterr string +} + +// Target variables for scanning into. +var ( + scanstr string + scanint int + scanint8 int8 + scanint16 int16 + scanint32 int32 + scanuint8 uint8 + scanuint16 uint16 +) + +var conversionTests = []conversionTest{ + // Exact conversions (destination pointer type matches source type) + {s: "foo", d: &scanstr, wantstr: "foo"}, + {s: 123, d: &scanint, wantint: 123}, + + // To strings + {s: []byte("byteslice"), d: &scanstr, wantstr: "byteslice"}, + {s: 123, d: &scanstr, wantstr: "123"}, + {s: int8(123), d: &scanstr, wantstr: "123"}, + {s: int64(123), d: &scanstr, wantstr: "123"}, + {s: uint8(123), d: &scanstr, wantstr: "123"}, + {s: uint16(123), d: &scanstr, wantstr: "123"}, + {s: uint32(123), d: &scanstr, wantstr: "123"}, + {s: uint64(123), d: &scanstr, wantstr: "123"}, + {s: 1.5, d: &scanstr, wantstr: "1.5"}, + + // Strings to integers + {s: "255", d: &scanuint8, wantuint: 255}, + {s: "256", d: &scanuint8, wanterr: `string "256" overflows uint8`}, + {s: "256", d: &scanuint16, wantuint: 256}, + {s: "-1", d: &scanint, wantint: -1}, + {s: "foo", d: &scanint, wanterr: `converting string "foo" to a int: parsing "foo": Invalid argument`}, +} + +func intValue(intptr interface{}) int64 { + return reflect.Indirect(reflect.ValueOf(intptr)).Int() +} + +func uintValue(intptr interface{}) uint64 { + return reflect.Indirect(reflect.ValueOf(intptr)).Uint() +} + +func TestConversions(t *testing.T) { + for n, ct := range conversionTests { + err := convertAssign(ct.d, ct.s) + errstr := "" + if err != nil { + errstr = err.String() + } + errf := func(format string, args ...interface{}) { + base := fmt.Sprintf("convertAssign #%d: for %v (%T) -> %T, ", n, ct.s, ct.s, ct.d) + t.Errorf(base+format, args...) + } + if errstr != ct.wanterr { + errf("got error %q, want error %q", errstr, ct.wanterr) + } + if ct.wantstr != "" && ct.wantstr != scanstr { + errf("want string %q, got %q", ct.wantstr, scanstr) + } + if ct.wantint != 0 && ct.wantint != intValue(ct.d) { + errf("want int %d, got %d", ct.wantint, intValue(ct.d)) + } + if ct.wantuint != 0 && ct.wantuint != uintValue(ct.d) { + errf("want uint %d, got %d", ct.wantuint, uintValue(ct.d)) + } + } +} + +func TestNullableString(t *testing.T) { + var ns NullableString + convertAssign(&ns, []byte("foo")) + if !ns.Valid { + t.Errorf("expecting not null") + } + if ns.String != "foo" { + t.Errorf("expecting foo; got %q", ns.String) + } + convertAssign(&ns, nil) + if ns.Valid { + t.Errorf("expecting null on nil") + } + if ns.String != "" { + t.Errorf("expecting blank on nil; got %q", ns.String) + } +} diff --git a/libgo/go/exp/sql/driver/driver.go b/libgo/go/exp/sql/driver/driver.go new file mode 100644 index 0000000..7508b19 --- /dev/null +++ b/libgo/go/exp/sql/driver/driver.go @@ -0,0 +1,169 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package driver defines interfaces to be implemented by database +// drivers as used by package sql. +// +// Code simply using databases should use package sql. +// +// Drivers only need to be aware of a subset of Go's types. The db package +// will convert all types into one of the following: +// +// int64 +// float64 +// bool +// nil +// []byte +// string [*] everywhere except from Rows.Next. +// +package driver + +import ( + "os" +) + +// Driver is the interface that must be implemented by a database +// driver. +type Driver interface { + // Open returns a new or cached connection to the database. + // The name is a string in a driver-specific format. + // + // The returned connection is only used by one goroutine at a + // time. + Open(name string) (Conn, os.Error) +} + +// Execer is an optional interface that may be implemented by a Driver +// or a Conn. +// +// If a Driver does not implement Execer, the sql package's DB.Exec +// method first obtains a free connection from its free pool or from +// the driver's Open method. Execer should only be implemented by +// drivers that can provide a more efficient implementation. +// +// If a Conn does not implement Execer, the db package's DB.Exec will +// first prepare a query, execute the statement, and then close the +// statement. +// +// All arguments are of a subset type as defined in the package docs. +type Execer interface { + Exec(query string, args []interface{}) (Result, os.Error) +} + +// Conn is a connection to a database. It is not used concurrently +// by multiple goroutines. +// +// Conn is assumed to be stateful. +type Conn interface { + // Prepare returns a prepared statement, bound to this connection. + Prepare(query string) (Stmt, os.Error) + + // Close invalidates and potentially stops any current + // prepared statements and transactions, marking this + // connection as no longer in use. The driver may cache or + // close its underlying connection to its database. + Close() os.Error + + // Begin starts and returns a new transaction. + Begin() (Tx, os.Error) +} + +// Result is the result of a query execution. +type Result interface { + // LastInsertId returns the database's auto-generated ID + // after, for example, an INSERT into a table with primary + // key. + LastInsertId() (int64, os.Error) + + // RowsAffected returns the number of rows affected by the + // query. + RowsAffected() (int64, os.Error) +} + +// Stmt is a prepared statement. It is bound to a Conn and not +// used by multiple goroutines concurrently. +type Stmt interface { + // Close closes the statement. + Close() os.Error + + // NumInput returns the number of placeholder parameters. + NumInput() int + + // Exec executes a query that doesn't return rows, such + // as an INSERT or UPDATE. The args are all of a subset + // type as defined above. + Exec(args []interface{}) (Result, os.Error) + + // Exec executes a query that may return rows, such as a + // SELECT. The args of all of a subset type as defined above. + Query(args []interface{}) (Rows, os.Error) +} + +// ColumnConverter may be optionally implemented by Stmt if the +// the statement is aware of its own columns' types and can +// convert from any type to a driver subset type. +type ColumnConverter interface { + // ColumnConverter returns a ValueConverter for the provided + // column index. If the type of a specific column isn't known + // or shouldn't be handled specially, DefaultValueConverter + // can be returned. + ColumnConverter(idx int) ValueConverter +} + +// Rows is an iterator over an executed query's results. +type Rows interface { + // Columns returns the names of the columns. The number of + // columns of the result is inferred from the length of the + // slice. If a particular column name isn't known, an empty + // string should be returned for that entry. + Columns() []string + + // Close closes the rows iterator. + Close() os.Error + + // Next is called to populate the next row of data into + // the provided slice. The provided slice will be the same + // size as the Columns() are wide. + // + // The dest slice may be populated with only with values + // of subset types defined above, but excluding string. + // All string values must be converted to []byte. + Next(dest []interface{}) os.Error +} + +// Tx is a transaction. +type Tx interface { + Commit() os.Error + Rollback() os.Error +} + +// RowsAffected implements Result for an INSERT or UPDATE operation +// which mutates a number of rows. +type RowsAffected int64 + +var _ Result = RowsAffected(0) + +func (RowsAffected) LastInsertId() (int64, os.Error) { + return 0, os.NewError("no LastInsertId available") +} + +func (v RowsAffected) RowsAffected() (int64, os.Error) { + return int64(v), nil +} + +// DDLSuccess is a pre-defined Result for drivers to return when a DDL +// command succeeds. +var DDLSuccess ddlSuccess + +type ddlSuccess struct{} + +var _ Result = ddlSuccess{} + +func (ddlSuccess) LastInsertId() (int64, os.Error) { + return 0, os.NewError("no LastInsertId available after DDL statement") +} + +func (ddlSuccess) RowsAffected() (int64, os.Error) { + return 0, os.NewError("no RowsAffected available after DDL statement") +} diff --git a/libgo/go/exp/sql/driver/types.go b/libgo/go/exp/sql/driver/types.go new file mode 100644 index 0000000..5521d53 --- /dev/null +++ b/libgo/go/exp/sql/driver/types.go @@ -0,0 +1,161 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package driver + +import ( + "fmt" + "os" + "reflect" + "strconv" +) + +// ValueConverter is the interface providing the ConvertValue method. +type ValueConverter interface { + // ConvertValue converts a value to a restricted subset type. + ConvertValue(v interface{}) (interface{}, os.Error) +} + +// Bool is a ValueConverter that converts input values to bools. +// +// The conversion rules are: +// - .... TODO(bradfitz): TBD +var Bool boolType + +type boolType struct{} + +var _ ValueConverter = boolType{} + +func (boolType) ConvertValue(v interface{}) (interface{}, os.Error) { + return nil, fmt.Errorf("TODO(bradfitz): bool conversions") +} + +// Int32 is a ValueConverter that converts input values to int64, +// respecting the limits of an int32 value. +var Int32 int32Type + +type int32Type struct{} + +var _ ValueConverter = int32Type{} + +func (int32Type) ConvertValue(v interface{}) (interface{}, os.Error) { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + i64 := rv.Int() + if i64 > (1<<31)-1 || i64 < -(1<<31) { + return nil, fmt.Errorf("sql/driver: value %d overflows int32", v) + } + return i64, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + u64 := rv.Uint() + if u64 > (1<<31)-1 { + return nil, fmt.Errorf("sql/driver: value %d overflows int32", v) + } + return int64(u64), nil + case reflect.String: + i, err := strconv.Atoi(rv.String()) + if err != nil { + return nil, fmt.Errorf("sql/driver: value %q can't be converted to int32", v) + } + return int64(i), nil + } + return nil, fmt.Errorf("sql/driver: unsupported value %v (type %T) converting to int32", v, v) +} + +// String is a ValueConverter that converts its input to a string. +// If the value is already a string or []byte, it's unchanged. +// If the value is of another type, conversion to string is done +// with fmt.Sprintf("%v", v). +var String stringType + +type stringType struct{} + +func (stringType) ConvertValue(v interface{}) (interface{}, os.Error) { + switch v.(type) { + case string, []byte: + return v, nil + } + return fmt.Sprintf("%v", v), nil +} + +// IsParameterSubsetType reports whether v is of a valid type for a +// parameter. These types are: +// +// int64 +// float64 +// bool +// nil +// []byte +// string +// +// This is the ame list as IsScanSubsetType, with the addition of +// string. +func IsParameterSubsetType(v interface{}) bool { + if IsScanSubsetType(v) { + return true + } + if _, ok := v.(string); ok { + return true + } + return false +} + +// IsScanSubsetType reports whether v is of a valid type for a +// value populated by Rows.Next. These types are: +// +// int64 +// float64 +// bool +// nil +// []byte +// +// This is the same list as IsParameterSubsetType, without string. +func IsScanSubsetType(v interface{}) bool { + if v == nil { + return true + } + switch v.(type) { + case int64, float64, []byte, bool: + return true + } + return false +} + +// DefaultParameterConverter is the default implementation of +// ValueConverter that's used when a Stmt doesn't implement +// ColumnConverter. +// +// DefaultParameterConverter returns the given value directly if +// IsSubsetType(value). Otherwise integer type are converted to +// int64, floats to float64, and strings to []byte. Other types are +// an error. +var DefaultParameterConverter defaultConverter + +type defaultConverter struct{} + +var _ ValueConverter = defaultConverter{} + +func (defaultConverter) ConvertValue(v interface{}) (interface{}, os.Error) { + if IsParameterSubsetType(v) { + return v, nil + } + + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: + return int64(rv.Uint()), nil + case reflect.Uint64: + u64 := rv.Uint() + if u64 >= 1<<63 { + return nil, fmt.Errorf("uint64 values with high bit set are not supported") + } + return int64(u64), nil + case reflect.Float32, reflect.Float64: + return rv.Float(), nil + } + return nil, fmt.Errorf("unsupported type %s", rv.Kind()) +} diff --git a/libgo/go/exp/sql/fakedb_test.go b/libgo/go/exp/sql/fakedb_test.go new file mode 100644 index 0000000..c906185 --- /dev/null +++ b/libgo/go/exp/sql/fakedb_test.go @@ -0,0 +1,497 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sql + +import ( + "fmt" + "log" + "os" + "strconv" + "strings" + "sync" + + "exp/sql/driver" +) + +var _ = log.Printf + +// fakeDriver is a fake database that implements Go's driver.Driver +// interface, just for testing. +// +// It speaks a query language that's semantically similar to but +// syntantically different and simpler than SQL. The syntax is as +// follows: +// +// WIPE +// CREATE|<tablename>|<col>=<type>,<col>=<type>,... +// where types are: "string", [u]int{8,16,32,64}, "bool" +// INSERT|<tablename>|col=val,col2=val2,col3=? +// SELECT|<tablename>|projectcol1,projectcol2|filtercol=?,filtercol2=? +// +// When opening a a fakeDriver's database, it starts empty with no +// tables. All tables and data are stored in memory only. +type fakeDriver struct { + mu sync.Mutex + openCount int + dbs map[string]*fakeDB +} + +type fakeDB struct { + name string + + mu sync.Mutex + free []*fakeConn + tables map[string]*table +} + +type table struct { + mu sync.Mutex + colname []string + coltype []string + rows []*row +} + +func (t *table) columnIndex(name string) int { + for n, nname := range t.colname { + if name == nname { + return n + } + } + return -1 +} + +type row struct { + cols []interface{} // must be same size as its table colname + coltype +} + +func (r *row) clone() *row { + nrow := &row{cols: make([]interface{}, len(r.cols))} + copy(nrow.cols, r.cols) + return nrow +} + +type fakeConn struct { + db *fakeDB // where to return ourselves to + + currTx *fakeTx +} + +type fakeTx struct { + c *fakeConn +} + +type fakeStmt struct { + c *fakeConn + q string // just for debugging + + cmd string + table string + + colName []string // used by CREATE, INSERT, SELECT (selected columns) + colType []string // used by CREATE + colValue []interface{} // used by INSERT (mix of strings and "?" for bound params) + placeholders int // used by INSERT/SELECT: number of ? params + + whereCol []string // used by SELECT (all placeholders) + + placeholderConverter []driver.ValueConverter // used by INSERT +} + +var fdriver driver.Driver = &fakeDriver{} + +func init() { + Register("test", fdriver) +} + +// Supports dsn forms: +// <dbname> +// <dbname>;wipe +func (d *fakeDriver) Open(dsn string) (driver.Conn, os.Error) { + d.mu.Lock() + defer d.mu.Unlock() + d.openCount++ + if d.dbs == nil { + d.dbs = make(map[string]*fakeDB) + } + parts := strings.Split(dsn, ";") + if len(parts) < 1 { + return nil, os.NewError("fakedb: no database name") + } + name := parts[0] + db, ok := d.dbs[name] + if !ok { + db = &fakeDB{name: name} + d.dbs[name] = db + } + return &fakeConn{db: db}, nil +} + +func (db *fakeDB) wipe() { + db.mu.Lock() + defer db.mu.Unlock() + db.tables = nil +} + +func (db *fakeDB) createTable(name string, columnNames, columnTypes []string) os.Error { + db.mu.Lock() + defer db.mu.Unlock() + if db.tables == nil { + db.tables = make(map[string]*table) + } + if _, exist := db.tables[name]; exist { + return fmt.Errorf("table %q already exists", name) + } + if len(columnNames) != len(columnTypes) { + return fmt.Errorf("create table of %q len(names) != len(types): %d vs %d", + name, len(columnNames), len(columnTypes)) + } + db.tables[name] = &table{colname: columnNames, coltype: columnTypes} + return nil +} + +// must be called with db.mu lock held +func (db *fakeDB) table(table string) (*table, bool) { + if db.tables == nil { + return nil, false + } + t, ok := db.tables[table] + return t, ok +} + +func (db *fakeDB) columnType(table, column string) (typ string, ok bool) { + db.mu.Lock() + defer db.mu.Unlock() + t, ok := db.table(table) + if !ok { + return + } + for n, cname := range t.colname { + if cname == column { + return t.coltype[n], true + } + } + return "", false +} + +func (c *fakeConn) Begin() (driver.Tx, os.Error) { + if c.currTx != nil { + return nil, os.NewError("already in a transaction") + } + c.currTx = &fakeTx{c: c} + return c.currTx, nil +} + +func (c *fakeConn) Close() os.Error { + if c.currTx != nil { + return os.NewError("can't close; in a Transaction") + } + if c.db == nil { + return os.NewError("can't close; already closed") + } + c.db = nil + return nil +} + +func errf(msg string, args ...interface{}) os.Error { + return os.NewError("fakedb: " + fmt.Sprintf(msg, args...)) +} + +// parts are table|selectCol1,selectCol2|whereCol=?,whereCol2=? +// (note that where where columns must always contain ? marks, +// just a limitation for fakedb) +func (c *fakeConn) prepareSelect(stmt *fakeStmt, parts []string) (driver.Stmt, os.Error) { + if len(parts) != 3 { + return nil, errf("invalid SELECT syntax with %d parts; want 3", len(parts)) + } + stmt.table = parts[0] + stmt.colName = strings.Split(parts[1], ",") + for n, colspec := range strings.Split(parts[2], ",") { + nameVal := strings.Split(colspec, "=") + if len(nameVal) != 2 { + return nil, errf("SELECT on table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n) + } + column, value := nameVal[0], nameVal[1] + _, ok := c.db.columnType(stmt.table, column) + if !ok { + return nil, errf("SELECT on table %q references non-existent column %q", stmt.table, column) + } + if value != "?" { + return nil, errf("SELECT on table %q has pre-bound value for where column %q; need a question mark", + stmt.table, column) + } + stmt.whereCol = append(stmt.whereCol, column) + stmt.placeholders++ + } + return stmt, nil +} + +// parts are table|col=type,col2=type2 +func (c *fakeConn) prepareCreate(stmt *fakeStmt, parts []string) (driver.Stmt, os.Error) { + if len(parts) != 2 { + return nil, errf("invalid CREATE syntax with %d parts; want 2", len(parts)) + } + stmt.table = parts[0] + for n, colspec := range strings.Split(parts[1], ",") { + nameType := strings.Split(colspec, "=") + if len(nameType) != 2 { + return nil, errf("CREATE table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n) + } + stmt.colName = append(stmt.colName, nameType[0]) + stmt.colType = append(stmt.colType, nameType[1]) + } + return stmt, nil +} + +// parts are table|col=?,col2=val +func (c *fakeConn) prepareInsert(stmt *fakeStmt, parts []string) (driver.Stmt, os.Error) { + if len(parts) != 2 { + return nil, errf("invalid INSERT syntax with %d parts; want 2", len(parts)) + } + stmt.table = parts[0] + for n, colspec := range strings.Split(parts[1], ",") { + nameVal := strings.Split(colspec, "=") + if len(nameVal) != 2 { + return nil, errf("INSERT table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n) + } + column, value := nameVal[0], nameVal[1] + ctype, ok := c.db.columnType(stmt.table, column) + if !ok { + return nil, errf("INSERT table %q references non-existent column %q", stmt.table, column) + } + stmt.colName = append(stmt.colName, column) + + if value != "?" { + var subsetVal interface{} + // Convert to driver subset type + switch ctype { + case "string": + subsetVal = []byte(value) + case "int32": + i, err := strconv.Atoi(value) + if err != nil { + return nil, errf("invalid conversion to int32 from %q", value) + } + subsetVal = int64(i) // int64 is a subset type, but not int32 + default: + return nil, errf("unsupported conversion for pre-bound parameter %q to type %q", value, ctype) + } + stmt.colValue = append(stmt.colValue, subsetVal) + } else { + stmt.placeholders++ + stmt.placeholderConverter = append(stmt.placeholderConverter, converterForType(ctype)) + stmt.colValue = append(stmt.colValue, "?") + } + } + return stmt, nil +} + +func (c *fakeConn) Prepare(query string) (driver.Stmt, os.Error) { + if c.db == nil { + panic("nil c.db; conn = " + fmt.Sprintf("%#v", c)) + } + parts := strings.Split(query, "|") + if len(parts) < 1 { + return nil, errf("empty query") + } + cmd := parts[0] + parts = parts[1:] + stmt := &fakeStmt{q: query, c: c, cmd: cmd} + switch cmd { + case "WIPE": + // Nothing + case "SELECT": + return c.prepareSelect(stmt, parts) + case "CREATE": + return c.prepareCreate(stmt, parts) + case "INSERT": + return c.prepareInsert(stmt, parts) + default: + return nil, errf("unsupported command type %q", cmd) + } + return stmt, nil +} + +func (s *fakeStmt) ColumnConverter(idx int) driver.ValueConverter { + return s.placeholderConverter[idx] +} + +func (s *fakeStmt) Close() os.Error { + return nil +} + +func (s *fakeStmt) Exec(args []interface{}) (driver.Result, os.Error) { + db := s.c.db + switch s.cmd { + case "WIPE": + db.wipe() + return driver.DDLSuccess, nil + case "CREATE": + if err := db.createTable(s.table, s.colName, s.colType); err != nil { + return nil, err + } + return driver.DDLSuccess, nil + case "INSERT": + return s.execInsert(args) + } + fmt.Printf("EXEC statement, cmd=%q: %#v\n", s.cmd, s) + return nil, fmt.Errorf("unimplemented statement Exec command type of %q", s.cmd) +} + +func (s *fakeStmt) execInsert(args []interface{}) (driver.Result, os.Error) { + db := s.c.db + if len(args) != s.placeholders { + panic("error in pkg db; should only get here if size is correct") + } + db.mu.Lock() + t, ok := db.table(s.table) + db.mu.Unlock() + if !ok { + return nil, fmt.Errorf("fakedb: table %q doesn't exist", s.table) + } + + t.mu.Lock() + defer t.mu.Unlock() + + cols := make([]interface{}, len(t.colname)) + argPos := 0 + for n, colname := range s.colName { + colidx := t.columnIndex(colname) + if colidx == -1 { + return nil, fmt.Errorf("fakedb: column %q doesn't exist or dropped since prepared statement was created", colname) + } + var val interface{} + if strvalue, ok := s.colValue[n].(string); ok && strvalue == "?" { + val = args[argPos] + argPos++ + } else { + val = s.colValue[n] + } + cols[colidx] = val + } + + t.rows = append(t.rows, &row{cols: cols}) + return driver.RowsAffected(1), nil +} + +func (s *fakeStmt) Query(args []interface{}) (driver.Rows, os.Error) { + db := s.c.db + if len(args) != s.placeholders { + panic("error in pkg db; should only get here if size is correct") + } + + db.mu.Lock() + t, ok := db.table(s.table) + db.mu.Unlock() + if !ok { + return nil, fmt.Errorf("fakedb: table %q doesn't exist", s.table) + } + t.mu.Lock() + defer t.mu.Unlock() + + colIdx := make(map[string]int) // select column name -> column index in table + for _, name := range s.colName { + idx := t.columnIndex(name) + if idx == -1 { + return nil, fmt.Errorf("fakedb: unknown column name %q", name) + } + colIdx[name] = idx + } + + mrows := []*row{} +rows: + for _, trow := range t.rows { + // Process the where clause, skipping non-match rows. This is lazy + // and just uses fmt.Sprintf("%v") to test equality. Good enough + // for test code. + for widx, wcol := range s.whereCol { + idx := t.columnIndex(wcol) + if idx == -1 { + return nil, fmt.Errorf("db: invalid where clause column %q", wcol) + } + tcol := trow.cols[idx] + if bs, ok := tcol.([]byte); ok { + // lazy hack to avoid sprintf %v on a []byte + tcol = string(bs) + } + if fmt.Sprintf("%v", tcol) != fmt.Sprintf("%v", args[widx]) { + continue rows + } + } + mrow := &row{cols: make([]interface{}, len(s.colName))} + for seli, name := range s.colName { + mrow.cols[seli] = trow.cols[colIdx[name]] + } + mrows = append(mrows, mrow) + } + + cursor := &rowsCursor{ + pos: -1, + rows: mrows, + cols: s.colName, + } + return cursor, nil +} + +func (s *fakeStmt) NumInput() int { + return s.placeholders +} + +func (tx *fakeTx) Commit() os.Error { + tx.c.currTx = nil + return nil +} + +func (tx *fakeTx) Rollback() os.Error { + tx.c.currTx = nil + return nil +} + +type rowsCursor struct { + cols []string + pos int + rows []*row + closed bool +} + +func (rc *rowsCursor) Close() os.Error { + rc.closed = true + return nil +} + +func (rc *rowsCursor) Columns() []string { + return rc.cols +} + +func (rc *rowsCursor) Next(dest []interface{}) os.Error { + if rc.closed { + return os.NewError("fakedb: cursor is closed") + } + rc.pos++ + if rc.pos >= len(rc.rows) { + return os.EOF // per interface spec + } + for i, v := range rc.rows[rc.pos].cols { + // TODO(bradfitz): convert to subset types? naah, I + // think the subset types should only be input to + // driver, but the db package should be able to handle + // a wider range of types coming out of drivers. all + // for ease of drivers, and to prevent drivers from + // messing up conversions or doing them differently. + dest[i] = v + } + return nil +} + +func converterForType(typ string) driver.ValueConverter { + switch typ { + case "bool": + return driver.Bool + case "int32": + return driver.Int32 + case "string": + return driver.String + } + panic("invalid fakedb column type of " + typ) +} diff --git a/libgo/go/exp/sql/sql.go b/libgo/go/exp/sql/sql.go new file mode 100644 index 0000000..7f0e0b2 --- /dev/null +++ b/libgo/go/exp/sql/sql.go @@ -0,0 +1,578 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sql provides a generic interface around SQL (or SQL-like) +// databases. +package sql + +import ( + "fmt" + "os" + "runtime" + "sync" + + "exp/sql/driver" +) + +var drivers = make(map[string]driver.Driver) + +// Register makes a database driver available by the provided name. +// If Register is called twice with the same name or if driver is nil, +// it panics. +func Register(name string, driver driver.Driver) { + if driver == nil { + panic("db: Register driver is nil") + } + if _, dup := drivers[name]; dup { + panic("db: Register called twice for driver " + name) + } + drivers[name] = driver +} + +// NullableString represents a string that may be null. +// NullableString implements the ScannerInto interface so +// it can be used as a scan destination: +// +// var s NullableString +// err := db.QueryRow("SELECT name FROM foo WHERE id=?", id).Scan(&s) +// ... +// if s.Valid { +// // use s.String +// } else { +// // NULL value +// } +// +// TODO(bradfitz): add other types. +type NullableString struct { + String string + Valid bool // Valid is true if String is not NULL +} + +// ScanInto implements the ScannerInto interface. +func (ms *NullableString) ScanInto(value interface{}) os.Error { + if value == nil { + ms.String, ms.Valid = "", false + return nil + } + ms.Valid = true + return convertAssign(&ms.String, value) +} + +// ScannerInto is an interface used by Scan. +type ScannerInto interface { + // ScanInto assigns a value from a database driver. + // + // The value will be of one of the following restricted + // set of types: + // + // int64 + // float64 + // bool + // []byte + // nil - for NULL values + // + // An error should be returned if the value can not be stored + // without loss of information. + ScanInto(value interface{}) os.Error +} + +// ErrNoRows is returned by Scan when QueryRow doesn't return a +// row. In such a case, QueryRow returns a placeholder *Row value that +// defers this error until a Scan. +var ErrNoRows = os.NewError("db: no rows in result set") + +// DB is a database handle. It's safe for concurrent use by multiple +// goroutines. +type DB struct { + driver driver.Driver + dsn string + + mu sync.Mutex + freeConn []driver.Conn +} + +// Open opens a database specified by its database driver name and a +// driver-specific data source name, usually consisting of at least a +// database name and connection information. +// +// Most users will open a database via a driver-specific connection +// helper function that returns a *DB. +func Open(driverName, dataSourceName string) (*DB, os.Error) { + driver, ok := drivers[driverName] + if !ok { + return nil, fmt.Errorf("db: unknown driver %q (forgotten import?)", driverName) + } + return &DB{driver: driver, dsn: dataSourceName}, nil +} + +func (db *DB) maxIdleConns() int { + const defaultMaxIdleConns = 2 + // TODO(bradfitz): ask driver, if supported, for its default preference + // TODO(bradfitz): let users override? + return defaultMaxIdleConns +} + +// conn returns a newly-opened or cached driver.Conn +func (db *DB) conn() (driver.Conn, os.Error) { + db.mu.Lock() + if n := len(db.freeConn); n > 0 { + conn := db.freeConn[n-1] + db.freeConn = db.freeConn[:n-1] + db.mu.Unlock() + return conn, nil + } + db.mu.Unlock() + return db.driver.Open(db.dsn) +} + +func (db *DB) connIfFree(wanted driver.Conn) (conn driver.Conn, ok bool) { + db.mu.Lock() + defer db.mu.Unlock() + for n, conn := range db.freeConn { + if conn == wanted { + db.freeConn[n] = db.freeConn[len(db.freeConn)-1] + db.freeConn = db.freeConn[:len(db.freeConn)-1] + return wanted, true + } + } + return nil, false +} + +func (db *DB) putConn(c driver.Conn) { + if n := len(db.freeConn); n < db.maxIdleConns() { + db.freeConn = append(db.freeConn, c) + return + } + db.closeConn(c) +} + +func (db *DB) closeConn(c driver.Conn) { + // TODO: check to see if we need this Conn for any prepared statements + // that are active. + c.Close() +} + +// Prepare creates a prepared statement for later execution. +func (db *DB) Prepare(query string) (*Stmt, os.Error) { + // TODO: check if db.driver supports an optional + // driver.Preparer interface and call that instead, if so, + // otherwise we make a prepared statement that's bound + // to a connection, and to execute this prepared statement + // we either need to use this connection (if it's free), else + // get a new connection + re-prepare + execute on that one. + ci, err := db.conn() + if err != nil { + return nil, err + } + defer db.putConn(ci) + si, err := ci.Prepare(query) + if err != nil { + return nil, err + } + stmt := &Stmt{ + db: db, + query: query, + css: []connStmt{{ci, si}}, + } + return stmt, nil +} + +// Exec executes a query without returning any rows. +func (db *DB) Exec(query string, args ...interface{}) (Result, os.Error) { + // Optional fast path, if the driver implements driver.Execer. + if execer, ok := db.driver.(driver.Execer); ok { + resi, err := execer.Exec(query, args) + if err != nil { + return nil, err + } + return result{resi}, nil + } + + // If the driver does not implement driver.Execer, we need + // a connection. + conn, err := db.conn() + if err != nil { + return nil, err + } + defer db.putConn(conn) + + if execer, ok := conn.(driver.Execer); ok { + resi, err := execer.Exec(query, args) + if err != nil { + return nil, err + } + return result{resi}, nil + } + + sti, err := conn.Prepare(query) + if err != nil { + return nil, err + } + defer sti.Close() + resi, err := sti.Exec(args) + if err != nil { + return nil, err + } + return result{resi}, nil +} + +// Query executes a query that returns rows, typically a SELECT. +func (db *DB) Query(query string, args ...interface{}) (*Rows, os.Error) { + stmt, err := db.Prepare(query) + if err != nil { + return nil, err + } + defer stmt.Close() + return stmt.Query(args...) +} + +// QueryRow executes a query that is expected to return at most one row. +// QueryRow always return a non-nil value. Errors are deferred until +// Row's Scan method is called. +func (db *DB) QueryRow(query string, args ...interface{}) *Row { + rows, err := db.Query(query, args...) + if err != nil { + return &Row{err: err} + } + return &Row{rows: rows} +} + +// Begin starts a transaction. The isolation level is dependent on +// the driver. +func (db *DB) Begin() (*Tx, os.Error) { + // TODO(bradfitz): add another method for beginning a transaction + // at a specific isolation level. + panic(todo()) +} + +// DriverDatabase returns the database's underlying driver. +func (db *DB) Driver() driver.Driver { + return db.driver +} + +// Tx is an in-progress database transaction. +type Tx struct { + +} + +// Commit commits the transaction. +func (tx *Tx) Commit() os.Error { + panic(todo()) +} + +// Rollback aborts the transaction. +func (tx *Tx) Rollback() os.Error { + panic(todo()) +} + +// Prepare creates a prepared statement. +func (tx *Tx) Prepare(query string) (*Stmt, os.Error) { + panic(todo()) +} + +// Exec executes a query that doesn't return rows. +// For example: an INSERT and UPDATE. +func (tx *Tx) Exec(query string, args ...interface{}) { + panic(todo()) +} + +// Query executes a query that returns rows, typically a SELECT. +func (tx *Tx) Query(query string, args ...interface{}) (*Rows, os.Error) { + panic(todo()) +} + +// QueryRow executes a query that is expected to return at most one row. +// QueryRow always return a non-nil value. Errors are deferred until +// Row's Scan method is called. +func (tx *Tx) QueryRow(query string, args ...interface{}) *Row { + panic(todo()) +} + +// connStmt is a prepared statement on a particular connection. +type connStmt struct { + ci driver.Conn + si driver.Stmt +} + +// Stmt is a prepared statement. Stmt is safe for concurrent use by multiple goroutines. +type Stmt struct { + // Immutable: + db *DB // where we came from + query string // that created the Sttm + + mu sync.Mutex + closed bool + css []connStmt // can use any that have idle connections +} + +func todo() string { + _, file, line, _ := runtime.Caller(1) + return fmt.Sprintf("%s:%d: TODO: implement", file, line) +} + +// Exec executes a prepared statement with the given arguments and +// returns a Result summarizing the effect of the statement. +func (s *Stmt) Exec(args ...interface{}) (Result, os.Error) { + ci, si, err := s.connStmt() + if err != nil { + return nil, err + } + defer s.db.putConn(ci) + + if want := si.NumInput(); len(args) != want { + return nil, fmt.Errorf("db: expected %d arguments, got %d", want, len(args)) + } + + // Convert args to subset types. + if cc, ok := si.(driver.ColumnConverter); ok { + for n, arg := range args { + args[n], err = cc.ColumnConverter(n).ConvertValue(arg) + if err != nil { + return nil, fmt.Errorf("db: converting Exec argument #%d's type: %v", n, err) + } + if !driver.IsParameterSubsetType(args[n]) { + return nil, fmt.Errorf("db: driver ColumnConverter error converted %T to unsupported type %T", + arg, args[n]) + } + } + } else { + for n, arg := range args { + args[n], err = driver.DefaultParameterConverter.ConvertValue(arg) + if err != nil { + return nil, fmt.Errorf("db: converting Exec argument #%d's type: %v", n, err) + } + } + } + + resi, err := si.Exec(args) + if err != nil { + return nil, err + } + return result{resi}, nil +} + +func (s *Stmt) connStmt(args ...interface{}) (driver.Conn, driver.Stmt, os.Error) { + s.mu.Lock() + if s.closed { + return nil, nil, os.NewError("db: statement is closed") + } + var cs connStmt + match := false + for _, v := range s.css { + // TODO(bradfitz): lazily clean up entries in this + // list with dead conns while enumerating + if _, match = s.db.connIfFree(cs.ci); match { + cs = v + break + } + } + s.mu.Unlock() + + // Make a new conn if all are busy. + // TODO(bradfitz): or wait for one? make configurable later? + if !match { + ci, err := s.db.conn() + if err != nil { + return nil, nil, err + } + si, err := ci.Prepare(s.query) + if err != nil { + return nil, nil, err + } + s.mu.Lock() + cs = connStmt{ci, si} + s.css = append(s.css, cs) + s.mu.Unlock() + } + + return cs.ci, cs.si, nil +} + +// Query executes a prepared query statement with the given arguments +// and returns the query results as a *Rows. +func (s *Stmt) Query(args ...interface{}) (*Rows, os.Error) { + ci, si, err := s.connStmt(args...) + if err != nil { + return nil, err + } + if len(args) != si.NumInput() { + return nil, fmt.Errorf("db: statement expects %d inputs; got %d", si.NumInput(), len(args)) + } + rowsi, err := si.Query(args) + if err != nil { + s.db.putConn(ci) + return nil, err + } + // Note: ownership of ci passes to the *Rows + rows := &Rows{ + db: s.db, + ci: ci, + rowsi: rowsi, + } + return rows, nil +} + +// QueryRow executes a prepared query statement with the given arguments. +// If an error occurs during the execution of the statement, that error will +// be returned by a call to Scan on the returned *Row, which is always non-nil. +// If the query selects no rows, the *Row's Scan will return ErrNoRows. +// Otherwise, the *Row's Scan scans the first selected row and discards +// the rest. +// +// Example usage: +// +// var name string +// err := nameByUseridStmt.QueryRow(id).Scan(&s) +func (s *Stmt) QueryRow(args ...interface{}) *Row { + rows, err := s.Query(args...) + if err != nil { + return &Row{err: err} + } + return &Row{rows: rows} +} + +// Close closes the statement. +func (s *Stmt) Close() os.Error { + s.mu.Lock() + defer s.mu.Unlock() // TODO(bradfitz): move this unlock after 'closed = true'? + if s.closed { + return nil + } + s.closed = true + for _, v := range s.css { + if ci, match := s.db.connIfFree(v.ci); match { + v.si.Close() + s.db.putConn(ci) + } else { + // TODO(bradfitz): care that we can't close + // this statement because the statement's + // connection is in use? + } + } + return nil +} + +// Rows is the result of a query. Its cursor starts before the first row +// of the result set. Use Next to advance through the rows: +// +// rows, err := db.Query("SELECT ...") +// ... +// for rows.Next() { +// var id int +// var name string +// err = rows.Scan(&id, &name) +// ... +// } +// err = rows.Error() // get any Error encountered during iteration +// ... +type Rows struct { + db *DB + ci driver.Conn // owned; must be returned when Rows is closed + rowsi driver.Rows + + closed bool + lastcols []interface{} + lasterr os.Error +} + +// Next prepares the next result row for reading with the Scan method. +// It returns true on success, false if there is no next result row. +// Every call to Scan, even the first one, must be preceded by a call +// to Next. +func (rs *Rows) Next() bool { + if rs.closed { + return false + } + if rs.lasterr != nil { + return false + } + if rs.lastcols == nil { + rs.lastcols = make([]interface{}, len(rs.rowsi.Columns())) + } + rs.lasterr = rs.rowsi.Next(rs.lastcols) + return rs.lasterr == nil +} + +// Error returns the error, if any, that was encountered during iteration. +func (rs *Rows) Error() os.Error { + if rs.lasterr == os.EOF { + return nil + } + return rs.lasterr +} + +// Scan copies the columns in the current row into the values pointed +// at by dest. If dest contains pointers to []byte, the slices should +// not be modified and should only be considered valid until the next +// call to Next or Scan. +func (rs *Rows) Scan(dest ...interface{}) os.Error { + if rs.closed { + return os.NewError("db: Rows closed") + } + if rs.lasterr != nil { + return rs.lasterr + } + if rs.lastcols == nil { + return os.NewError("db: Scan called without calling Next") + } + if len(dest) != len(rs.lastcols) { + return fmt.Errorf("db: expected %d destination arguments in Scan, not %d", len(rs.lastcols), len(dest)) + } + for i, sv := range rs.lastcols { + err := convertAssign(dest[i], sv) + if err != nil { + return fmt.Errorf("db: Scan error on column index %d: %v", i, err) + } + } + return nil +} + +// Close closes the Rows, preventing further enumeration. If the +// end is encountered, the Rows are closed automatically. Close +// is idempotent. +func (rs *Rows) Close() os.Error { + if rs.closed { + return nil + } + rs.closed = true + err := rs.rowsi.Close() + rs.db.putConn(rs.ci) + return err +} + +// Row is the result of calling QueryRow to select a single row. +type Row struct { + // One of these two will be non-nil: + err os.Error // deferred error for easy chaining + rows *Rows +} + +// Scan copies the columns from the matched row into the values +// pointed at by dest. If more than one row matches the query, +// Scan uses the first row and discards the rest. If no row matches +// the query, Scan returns ErrNoRows. +// +// If dest contains pointers to []byte, the slices should not be +// modified and should only be considered valid until the next call to +// Next or Scan. +func (r *Row) Scan(dest ...interface{}) os.Error { + if r.err != nil { + return r.err + } + defer r.rows.Close() + if !r.rows.Next() { + return ErrNoRows + } + return r.rows.Scan(dest...) +} + +// A Result summarizes an executed SQL command. +type Result interface { + LastInsertId() (int64, os.Error) + RowsAffected() (int64, os.Error) +} + +type result struct { + driver.Result +} diff --git a/libgo/go/exp/sql/sql_test.go b/libgo/go/exp/sql/sql_test.go new file mode 100644 index 0000000..eaa0a90 --- /dev/null +++ b/libgo/go/exp/sql/sql_test.go @@ -0,0 +1,145 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sql + +import ( + "strings" + "testing" +) + +func newTestDB(t *testing.T, name string) *DB { + db, err := Open("test", "foo") + if err != nil { + t.Fatalf("Open: %v", err) + } + if _, err := db.Exec("WIPE"); err != nil { + t.Fatalf("exec wipe: %v", err) + } + if name == "people" { + exec(t, db, "CREATE|people|name=string,age=int32,dead=bool") + exec(t, db, "INSERT|people|name=Alice,age=?", 1) + exec(t, db, "INSERT|people|name=Bob,age=?", 2) + exec(t, db, "INSERT|people|name=Chris,age=?", 3) + + } + return db +} + +func exec(t *testing.T, db *DB, query string, args ...interface{}) { + _, err := db.Exec(query, args...) + if err != nil { + t.Fatalf("Exec of %q: %v", query, err) + } +} + +func TestQuery(t *testing.T) { + db := newTestDB(t, "people") + var name string + var age int + + err := db.QueryRow("SELECT|people|age,name|age=?", 3).Scan(&age) + if err == nil || !strings.Contains(err.String(), "expected 2 destination arguments") { + t.Errorf("expected error from wrong number of arguments; actually got: %v", err) + } + + err = db.QueryRow("SELECT|people|age,name|age=?", 2).Scan(&age, &name) + if err != nil { + t.Fatalf("age QueryRow+Scan: %v", err) + } + if name != "Bob" { + t.Errorf("expected name Bob, got %q", name) + } + if age != 2 { + t.Errorf("expected age 2, got %d", age) + } + + err = db.QueryRow("SELECT|people|age,name|name=?", "Alice").Scan(&age, &name) + if err != nil { + t.Fatalf("name QueryRow+Scan: %v", err) + } + if name != "Alice" { + t.Errorf("expected name Alice, got %q", name) + } + if age != 1 { + t.Errorf("expected age 1, got %d", age) + } +} + +func TestStatementQueryRow(t *testing.T) { + db := newTestDB(t, "people") + stmt, err := db.Prepare("SELECT|people|age|name=?") + if err != nil { + t.Fatalf("Prepare: %v", err) + } + var age int + for n, tt := range []struct { + name string + want int + }{ + {"Alice", 1}, + {"Bob", 2}, + {"Chris", 3}, + } { + if err := stmt.QueryRow(tt.name).Scan(&age); err != nil { + t.Errorf("%d: on %q, QueryRow/Scan: %v", n, tt.name, err) + } else if age != tt.want { + t.Errorf("%d: age=%d, want %d", n, age, tt.want) + } + } + +} + +// just a test of fakedb itself +func TestBogusPreboundParameters(t *testing.T) { + db := newTestDB(t, "foo") + exec(t, db, "CREATE|t1|name=string,age=int32,dead=bool") + _, err := db.Prepare("INSERT|t1|name=?,age=bogusconversion") + if err == nil { + t.Fatalf("expected error") + } + if err.String() != `fakedb: invalid conversion to int32 from "bogusconversion"` { + t.Errorf("unexpected error: %v", err) + } +} + +func TestDb(t *testing.T) { + db := newTestDB(t, "foo") + exec(t, db, "CREATE|t1|name=string,age=int32,dead=bool") + stmt, err := db.Prepare("INSERT|t1|name=?,age=?") + if err != nil { + t.Errorf("Stmt, err = %v, %v", stmt, err) + } + + type execTest struct { + args []interface{} + wantErr string + } + execTests := []execTest{ + // Okay: + {[]interface{}{"Brad", 31}, ""}, + {[]interface{}{"Brad", int64(31)}, ""}, + {[]interface{}{"Bob", "32"}, ""}, + {[]interface{}{7, 9}, ""}, + + // Invalid conversions: + {[]interface{}{"Brad", int64(0xFFFFFFFF)}, "db: converting Exec argument #1's type: sql/driver: value 4294967295 overflows int32"}, + {[]interface{}{"Brad", "strconv fail"}, "db: converting Exec argument #1's type: sql/driver: value \"strconv fail\" can't be converted to int32"}, + + // Wrong number of args: + {[]interface{}{}, "db: expected 2 arguments, got 0"}, + {[]interface{}{1, 2, 3}, "db: expected 2 arguments, got 3"}, + } + for n, et := range execTests { + _, err := stmt.Exec(et.args...) + errStr := "" + if err != nil { + errStr = err.String() + } + if errStr != et.wantErr { + t.Errorf("stmt.Execute #%d: for %v, got error %q, want error %q", + n, et.args, errStr, et.wantErr) + } + } +} diff --git a/libgo/go/exp/ssh/channel.go b/libgo/go/exp/ssh/channel.go new file mode 100644 index 0000000..922584f --- /dev/null +++ b/libgo/go/exp/ssh/channel.go @@ -0,0 +1,317 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "os" + "sync" +) + +// A Channel is an ordered, reliable, duplex stream that is multiplexed over an +// SSH connection. +type Channel interface { + // Accept accepts the channel creation request. + Accept() os.Error + // Reject rejects the channel creation request. After calling this, no + // other methods on the Channel may be called. If they are then the + // peer is likely to signal a protocol error and drop the connection. + Reject(reason RejectionReason, message string) os.Error + + // Read may return a ChannelRequest as an os.Error. + Read(data []byte) (int, os.Error) + Write(data []byte) (int, os.Error) + Close() os.Error + + // AckRequest either sends an ack or nack to the channel request. + AckRequest(ok bool) os.Error + + // ChannelType returns the type of the channel, as supplied by the + // client. + ChannelType() string + // ExtraData returns the arbitary payload for this channel, as supplied + // by the client. This data is specific to the channel type. + ExtraData() []byte +} + +// ChannelRequest represents a request sent on a channel, outside of the normal +// stream of bytes. It may result from calling Read on a Channel. +type ChannelRequest struct { + Request string + WantReply bool + Payload []byte +} + +func (c ChannelRequest) String() string { + return "channel request received" +} + +// RejectionReason is an enumeration used when rejecting channel creation +// requests. See RFC 4254, section 5.1. +type RejectionReason int + +const ( + Prohibited RejectionReason = iota + 1 + ConnectionFailed + UnknownChannelType + ResourceShortage +) + +type channel struct { + // immutable once created + chanType string + extraData []byte + + theyClosed bool + theySentEOF bool + weClosed bool + dead bool + + serverConn *ServerConnection + myId, theirId uint32 + myWindow, theirWindow uint32 + maxPacketSize uint32 + err os.Error + + pendingRequests []ChannelRequest + pendingData []byte + head, length int + + // This lock is inferior to serverConn.lock + lock sync.Mutex + cond *sync.Cond +} + +func (c *channel) Accept() os.Error { + c.serverConn.lock.Lock() + defer c.serverConn.lock.Unlock() + + if c.serverConn.err != nil { + return c.serverConn.err + } + + confirm := channelOpenConfirmMsg{ + PeersId: c.theirId, + MyId: c.myId, + MyWindow: c.myWindow, + MaxPacketSize: c.maxPacketSize, + } + return c.serverConn.writePacket(marshal(msgChannelOpenConfirm, confirm)) +} + +func (c *channel) Reject(reason RejectionReason, message string) os.Error { + c.serverConn.lock.Lock() + defer c.serverConn.lock.Unlock() + + if c.serverConn.err != nil { + return c.serverConn.err + } + + reject := channelOpenFailureMsg{ + PeersId: c.theirId, + Reason: uint32(reason), + Message: message, + Language: "en", + } + return c.serverConn.writePacket(marshal(msgChannelOpenFailure, reject)) +} + +func (c *channel) handlePacket(packet interface{}) { + c.lock.Lock() + defer c.lock.Unlock() + + switch packet := packet.(type) { + case *channelRequestMsg: + req := ChannelRequest{ + Request: packet.Request, + WantReply: packet.WantReply, + Payload: packet.RequestSpecificData, + } + + c.pendingRequests = append(c.pendingRequests, req) + c.cond.Signal() + case *channelCloseMsg: + c.theyClosed = true + c.cond.Signal() + case *channelEOFMsg: + c.theySentEOF = true + c.cond.Signal() + default: + panic("unknown packet type") + } +} + +func (c *channel) handleData(data []byte) { + c.lock.Lock() + defer c.lock.Unlock() + + // The other side should never send us more than our window. + if len(data)+c.length > len(c.pendingData) { + // TODO(agl): we should tear down the channel with a protocol + // error. + return + } + + c.myWindow -= uint32(len(data)) + for i := 0; i < 2; i++ { + tail := c.head + c.length + if tail > len(c.pendingData) { + tail -= len(c.pendingData) + } + n := copy(c.pendingData[tail:], data) + data = data[n:] + c.length += n + } + + c.cond.Signal() +} + +func (c *channel) Read(data []byte) (n int, err os.Error) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.err != nil { + return 0, c.err + } + + if c.myWindow <= uint32(len(c.pendingData))/2 { + packet := marshal(msgChannelWindowAdjust, windowAdjustMsg{ + PeersId: c.theirId, + AdditionalBytes: uint32(len(c.pendingData)) - c.myWindow, + }) + if err := c.serverConn.writePacket(packet); err != nil { + return 0, err + } + } + + for { + if c.theySentEOF || c.theyClosed || c.dead { + return 0, os.EOF + } + + if len(c.pendingRequests) > 0 { + req := c.pendingRequests[0] + if len(c.pendingRequests) == 1 { + c.pendingRequests = nil + } else { + oldPendingRequests := c.pendingRequests + c.pendingRequests = make([]ChannelRequest, len(oldPendingRequests)-1) + copy(c.pendingRequests, oldPendingRequests[1:]) + } + + return 0, req + } + + if c.length > 0 { + tail := c.head + c.length + if tail > len(c.pendingData) { + tail -= len(c.pendingData) + } + n = copy(data, c.pendingData[c.head:tail]) + c.head += n + c.length -= n + if c.head == len(c.pendingData) { + c.head = 0 + } + return + } + + c.cond.Wait() + } + + panic("unreachable") +} + +func (c *channel) Write(data []byte) (n int, err os.Error) { + for len(data) > 0 { + c.lock.Lock() + if c.dead || c.weClosed { + return 0, os.EOF + } + + if c.theirWindow == 0 { + c.cond.Wait() + continue + } + c.lock.Unlock() + + todo := data + if uint32(len(todo)) > c.theirWindow { + todo = todo[:c.theirWindow] + } + + packet := make([]byte, 1+4+4+len(todo)) + packet[0] = msgChannelData + packet[1] = byte(c.theirId) >> 24 + packet[2] = byte(c.theirId) >> 16 + packet[3] = byte(c.theirId) >> 8 + packet[4] = byte(c.theirId) + packet[5] = byte(len(todo)) >> 24 + packet[6] = byte(len(todo)) >> 16 + packet[7] = byte(len(todo)) >> 8 + packet[8] = byte(len(todo)) + copy(packet[9:], todo) + + c.serverConn.lock.Lock() + if err = c.serverConn.writePacket(packet); err != nil { + c.serverConn.lock.Unlock() + return + } + c.serverConn.lock.Unlock() + + n += len(todo) + data = data[len(todo):] + } + + return +} + +func (c *channel) Close() os.Error { + c.serverConn.lock.Lock() + defer c.serverConn.lock.Unlock() + + if c.serverConn.err != nil { + return c.serverConn.err + } + + if c.weClosed { + return os.NewError("ssh: channel already closed") + } + c.weClosed = true + + closeMsg := channelCloseMsg{ + PeersId: c.theirId, + } + return c.serverConn.writePacket(marshal(msgChannelClose, closeMsg)) +} + +func (c *channel) AckRequest(ok bool) os.Error { + c.serverConn.lock.Lock() + defer c.serverConn.lock.Unlock() + + if c.serverConn.err != nil { + return c.serverConn.err + } + + if ok { + ack := channelRequestSuccessMsg{ + PeersId: c.theirId, + } + return c.serverConn.writePacket(marshal(msgChannelSuccess, ack)) + } else { + ack := channelRequestFailureMsg{ + PeersId: c.theirId, + } + return c.serverConn.writePacket(marshal(msgChannelFailure, ack)) + } + panic("unreachable") +} + +func (c *channel) ChannelType() string { + return c.chanType +} + +func (c *channel) ExtraData() []byte { + return c.extraData +} diff --git a/libgo/go/exp/ssh/common.go b/libgo/go/exp/ssh/common.go new file mode 100644 index 0000000..739bd2f --- /dev/null +++ b/libgo/go/exp/ssh/common.go @@ -0,0 +1,129 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "big" + "strconv" + "sync" +) + +// These are string constants in the SSH protocol. +const ( + kexAlgoDH14SHA1 = "diffie-hellman-group14-sha1" + hostAlgoRSA = "ssh-rsa" + cipherAES128CTR = "aes128-ctr" + macSHA196 = "hmac-sha1-96" + compressionNone = "none" + serviceUserAuth = "ssh-userauth" + serviceSSH = "ssh-connection" +) + +var supportedKexAlgos = []string{kexAlgoDH14SHA1} +var supportedHostKeyAlgos = []string{hostAlgoRSA} +var supportedCiphers = []string{cipherAES128CTR} +var supportedMACs = []string{macSHA196} +var supportedCompressions = []string{compressionNone} + +// dhGroup is a multiplicative group suitable for implementing Diffie-Hellman key agreement. +type dhGroup struct { + g, p *big.Int +} + +// dhGroup14 is the group called diffie-hellman-group14-sha1 in RFC 4253 and +// Oakley Group 14 in RFC 3526. +var dhGroup14 *dhGroup + +var dhGroup14Once sync.Once + +func initDHGroup14() { + p, _ := new(big.Int).SetString("FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF", 16) + + dhGroup14 = &dhGroup{ + g: new(big.Int).SetInt64(2), + p: p, + } +} + +// UnexpectedMessageError results when the SSH message that we received didn't +// match what we wanted. +type UnexpectedMessageError struct { + expected, got uint8 +} + +func (u UnexpectedMessageError) String() string { + return "ssh: unexpected message type " + strconv.Itoa(int(u.got)) + " (expected " + strconv.Itoa(int(u.expected)) + ")" +} + +// ParseError results from a malformed SSH message. +type ParseError struct { + msgType uint8 +} + +func (p ParseError) String() string { + return "ssh: parse error in message type " + strconv.Itoa(int(p.msgType)) +} + +type handshakeMagics struct { + clientVersion, serverVersion []byte + clientKexInit, serverKexInit []byte +} + +func findCommonAlgorithm(clientAlgos []string, serverAlgos []string) (commonAlgo string, ok bool) { + for _, clientAlgo := range clientAlgos { + for _, serverAlgo := range serverAlgos { + if clientAlgo == serverAlgo { + return clientAlgo, true + } + } + } + + return +} + +func findAgreedAlgorithms(transport *transport, clientKexInit, serverKexInit *kexInitMsg) (kexAlgo, hostKeyAlgo string, ok bool) { + kexAlgo, ok = findCommonAlgorithm(clientKexInit.KexAlgos, serverKexInit.KexAlgos) + if !ok { + return + } + + hostKeyAlgo, ok = findCommonAlgorithm(clientKexInit.ServerHostKeyAlgos, serverKexInit.ServerHostKeyAlgos) + if !ok { + return + } + + transport.writer.cipherAlgo, ok = findCommonAlgorithm(clientKexInit.CiphersClientServer, serverKexInit.CiphersClientServer) + if !ok { + return + } + + transport.reader.cipherAlgo, ok = findCommonAlgorithm(clientKexInit.CiphersServerClient, serverKexInit.CiphersServerClient) + if !ok { + return + } + + transport.writer.macAlgo, ok = findCommonAlgorithm(clientKexInit.MACsClientServer, serverKexInit.MACsClientServer) + if !ok { + return + } + + transport.reader.macAlgo, ok = findCommonAlgorithm(clientKexInit.MACsServerClient, serverKexInit.MACsServerClient) + if !ok { + return + } + + transport.writer.compressionAlgo, ok = findCommonAlgorithm(clientKexInit.CompressionClientServer, serverKexInit.CompressionClientServer) + if !ok { + return + } + + transport.reader.compressionAlgo, ok = findCommonAlgorithm(clientKexInit.CompressionServerClient, serverKexInit.CompressionServerClient) + if !ok { + return + } + + ok = true + return +} diff --git a/libgo/go/exp/ssh/doc.go b/libgo/go/exp/ssh/doc.go new file mode 100644 index 0000000..54a7ba9 --- /dev/null +++ b/libgo/go/exp/ssh/doc.go @@ -0,0 +1,79 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package ssh implements an SSH server. + +SSH is a transport security protocol, an authentication protocol and a +family of application protocols. The most typical application level +protocol is a remote shell and this is specifically implemented. However, +the multiplexed nature of SSH is exposed to users that wish to support +others. + +An SSH server is represented by a Server, which manages a number of +ServerConnections and handles authentication. + + var s Server + s.PubKeyCallback = pubKeyAuth + s.PasswordCallback = passwordAuth + + pemBytes, err := ioutil.ReadFile("id_rsa") + if err != nil { + panic("Failed to load private key") + } + err = s.SetRSAPrivateKey(pemBytes) + if err != nil { + panic("Failed to parse private key") + } + +Once a Server has been set up, connections can be attached. + + var sConn ServerConnection + sConn.Server = &s + err = sConn.Handshake(conn) + if err != nil { + panic("failed to handshake") + } + +An SSH connection multiplexes several channels, which must be accepted themselves: + + + for { + channel, err := sConn.Accept() + if err != nil { + panic("error from Accept") + } + + ... + } + +Accept reads from the connection, demultiplexes packets to their corresponding +channels and returns when a new channel request is seen. Some goroutine must +always be calling Accept; otherwise no messages will be forwarded to the +channels. + +Channels have a type, depending on the application level protocol intended. In +the case of a shell, the type is "session" and ServerShell may be used to +present a simple terminal interface. + + if channel.ChannelType() != "session" { + c.Reject(UnknownChannelType, "unknown channel type") + return + } + channel.Accept() + + shell := NewServerShell(channel, "> ") + go func() { + defer channel.Close() + for { + line, err := shell.ReadLine() + if err != nil { + break + } + println(line) + } + return + }() +*/ +package ssh diff --git a/libgo/go/exp/ssh/messages.go b/libgo/go/exp/ssh/messages.go new file mode 100644 index 0000000..1d0bc57 --- /dev/null +++ b/libgo/go/exp/ssh/messages.go @@ -0,0 +1,636 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "big" + "bytes" + "io" + "os" + "reflect" +) + +// These are SSH message type numbers. They are scattered around several +// documents but many were taken from +// http://www.iana.org/assignments/ssh-parameters/ssh-parameters.xml#ssh-parameters-1 +const ( + msgDisconnect = 1 + msgIgnore = 2 + msgUnimplemented = 3 + msgDebug = 4 + msgServiceRequest = 5 + msgServiceAccept = 6 + + msgKexInit = 20 + msgNewKeys = 21 + + msgKexDHInit = 30 + msgKexDHReply = 31 + + msgUserAuthRequest = 50 + msgUserAuthFailure = 51 + msgUserAuthSuccess = 52 + msgUserAuthBanner = 53 + msgUserAuthPubKeyOk = 60 + + msgGlobalRequest = 80 + msgRequestSuccess = 81 + msgRequestFailure = 82 + + msgChannelOpen = 90 + msgChannelOpenConfirm = 91 + msgChannelOpenFailure = 92 + msgChannelWindowAdjust = 93 + msgChannelData = 94 + msgChannelExtendedData = 95 + msgChannelEOF = 96 + msgChannelClose = 97 + msgChannelRequest = 98 + msgChannelSuccess = 99 + msgChannelFailure = 100 +) + +// SSH messages: +// +// These structures mirror the wire format of the corresponding SSH messages. +// They are marshaled using reflection with the marshal and unmarshal functions +// in this file. The only wrinkle is that a final member of type []byte with a +// ssh tag of "rest" receives the remainder of a packet when unmarshaling. + +// See RFC 4253, section 11.1. +type disconnectMsg struct { + Reason uint32 + Message string + Language string +} + +// See RFC 4253, section 7.1. +type kexInitMsg struct { + Cookie [16]byte + KexAlgos []string + ServerHostKeyAlgos []string + CiphersClientServer []string + CiphersServerClient []string + MACsClientServer []string + MACsServerClient []string + CompressionClientServer []string + CompressionServerClient []string + LanguagesClientServer []string + LanguagesServerClient []string + FirstKexFollows bool + Reserved uint32 +} + +// See RFC 4253, section 8. +type kexDHInitMsg struct { + X *big.Int +} + +type kexDHReplyMsg struct { + HostKey []byte + Y *big.Int + Signature []byte +} + +// See RFC 4253, section 10. +type serviceRequestMsg struct { + Service string +} + +// See RFC 4253, section 10. +type serviceAcceptMsg struct { + Service string +} + +// See RFC 4252, section 5. +type userAuthRequestMsg struct { + User string + Service string + Method string + Payload []byte `ssh:"rest"` +} + +// See RFC 4252, section 5.1 +type userAuthFailureMsg struct { + Methods []string + PartialSuccess bool +} + +// See RFC 4254, section 5.1. +type channelOpenMsg struct { + ChanType string + PeersId uint32 + PeersWindow uint32 + MaxPacketSize uint32 + TypeSpecificData []byte `ssh:"rest"` +} + +// See RFC 4254, section 5.1. +type channelOpenConfirmMsg struct { + PeersId uint32 + MyId uint32 + MyWindow uint32 + MaxPacketSize uint32 + TypeSpecificData []byte `ssh:"rest"` +} + +// See RFC 4254, section 5.1. +type channelOpenFailureMsg struct { + PeersId uint32 + Reason uint32 + Message string + Language string +} + +// See RFC 4254, section 5.2. +type channelData struct { + PeersId uint32 + Payload []byte `ssh:"rest"` +} + +// See RFC 4254, section 5.2. +type channelExtendedData struct { + PeersId uint32 + Datatype uint32 + Data string +} + +type channelRequestMsg struct { + PeersId uint32 + Request string + WantReply bool + RequestSpecificData []byte `ssh:"rest"` +} + +// See RFC 4254, section 5.4. +type channelRequestSuccessMsg struct { + PeersId uint32 +} + +// See RFC 4254, section 5.4. +type channelRequestFailureMsg struct { + PeersId uint32 +} + +// See RFC 4254, section 5.3 +type channelCloseMsg struct { + PeersId uint32 +} + +// See RFC 4254, section 5.3 +type channelEOFMsg struct { + PeersId uint32 +} + +// See RFC 4254, section 4 +type globalRequestMsg struct { + Type string + WantReply bool +} + +// See RFC 4254, section 5.2 +type windowAdjustMsg struct { + PeersId uint32 + AdditionalBytes uint32 +} + +// See RFC 4252, section 7 +type userAuthPubKeyOkMsg struct { + Algo string + PubKey string +} + +// unmarshal parses the SSH wire data in packet into out using reflection. +// expectedType is the expected SSH message type. It either returns nil on +// success, or a ParseError or UnexpectedMessageError on error. +func unmarshal(out interface{}, packet []byte, expectedType uint8) os.Error { + if len(packet) == 0 { + return ParseError{expectedType} + } + if packet[0] != expectedType { + return UnexpectedMessageError{expectedType, packet[0]} + } + packet = packet[1:] + + v := reflect.ValueOf(out).Elem() + structType := v.Type() + var ok bool + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + t := field.Type() + switch t.Kind() { + case reflect.Bool: + if len(packet) < 1 { + return ParseError{expectedType} + } + field.SetBool(packet[0] != 0) + packet = packet[1:] + case reflect.Array: + if t.Elem().Kind() != reflect.Uint8 { + panic("array of non-uint8") + } + if len(packet) < t.Len() { + return ParseError{expectedType} + } + for j := 0; j < t.Len(); j++ { + field.Index(j).Set(reflect.ValueOf(packet[j])) + } + packet = packet[t.Len():] + case reflect.Uint32: + var u32 uint32 + if u32, packet, ok = parseUint32(packet); !ok { + return ParseError{expectedType} + } + field.SetUint(uint64(u32)) + case reflect.String: + var s []byte + if s, packet, ok = parseString(packet); !ok { + return ParseError{expectedType} + } + field.SetString(string(s)) + case reflect.Slice: + switch t.Elem().Kind() { + case reflect.Uint8: + if structType.Field(i).Tag.Get("ssh") == "rest" { + field.Set(reflect.ValueOf(packet)) + packet = nil + } else { + var s []byte + if s, packet, ok = parseString(packet); !ok { + return ParseError{expectedType} + } + field.Set(reflect.ValueOf(s)) + } + case reflect.String: + var nl []string + if nl, packet, ok = parseNameList(packet); !ok { + return ParseError{expectedType} + } + field.Set(reflect.ValueOf(nl)) + default: + panic("slice of unknown type") + } + case reflect.Ptr: + if t == bigIntType { + var n *big.Int + if n, packet, ok = parseInt(packet); !ok { + return ParseError{expectedType} + } + field.Set(reflect.ValueOf(n)) + } else { + panic("pointer to unknown type") + } + default: + panic("unknown type") + } + } + + if len(packet) != 0 { + return ParseError{expectedType} + } + + return nil +} + +// marshal serializes the message in msg, using the given message type. +func marshal(msgType uint8, msg interface{}) []byte { + var out []byte + out = append(out, msgType) + + v := reflect.ValueOf(msg) + structType := v.Type() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + t := field.Type() + switch t.Kind() { + case reflect.Bool: + var v uint8 + if field.Bool() { + v = 1 + } + out = append(out, v) + case reflect.Array: + if t.Elem().Kind() != reflect.Uint8 { + panic("array of non-uint8") + } + for j := 0; j < t.Len(); j++ { + out = append(out, byte(field.Index(j).Uint())) + } + case reflect.Uint32: + u32 := uint32(field.Uint()) + out = append(out, byte(u32>>24)) + out = append(out, byte(u32>>16)) + out = append(out, byte(u32>>8)) + out = append(out, byte(u32)) + case reflect.String: + s := field.String() + out = append(out, byte(len(s)>>24)) + out = append(out, byte(len(s)>>16)) + out = append(out, byte(len(s)>>8)) + out = append(out, byte(len(s))) + out = append(out, s...) + case reflect.Slice: + switch t.Elem().Kind() { + case reflect.Uint8: + length := field.Len() + if structType.Field(i).Tag.Get("ssh") != "rest" { + out = append(out, byte(length>>24)) + out = append(out, byte(length>>16)) + out = append(out, byte(length>>8)) + out = append(out, byte(length)) + } + for j := 0; j < length; j++ { + out = append(out, byte(field.Index(j).Uint())) + } + case reflect.String: + var length int + for j := 0; j < field.Len(); j++ { + if j != 0 { + length++ /* comma */ + } + length += len(field.Index(j).String()) + } + + out = append(out, byte(length>>24)) + out = append(out, byte(length>>16)) + out = append(out, byte(length>>8)) + out = append(out, byte(length)) + for j := 0; j < field.Len(); j++ { + if j != 0 { + out = append(out, ',') + } + out = append(out, field.Index(j).String()...) + } + default: + panic("slice of unknown type") + } + case reflect.Ptr: + if t == bigIntType { + var n *big.Int + nValue := reflect.ValueOf(&n) + nValue.Elem().Set(field) + needed := intLength(n) + oldLength := len(out) + + if cap(out)-len(out) < needed { + newOut := make([]byte, len(out), 2*(len(out)+needed)) + copy(newOut, out) + out = newOut + } + out = out[:oldLength+needed] + marshalInt(out[oldLength:], n) + } else { + panic("pointer to unknown type") + } + } + } + + return out +} + +var bigOne = big.NewInt(1) + +func parseString(in []byte) (out, rest []byte, ok bool) { + if len(in) < 4 { + return + } + length := uint32(in[0])<<24 | uint32(in[1])<<16 | uint32(in[2])<<8 | uint32(in[3]) + if uint32(len(in)) < 4+length { + return + } + out = in[4 : 4+length] + rest = in[4+length:] + ok = true + return +} + +var comma = []byte{','} + +func parseNameList(in []byte) (out []string, rest []byte, ok bool) { + contents, rest, ok := parseString(in) + if !ok { + return + } + if len(contents) == 0 { + return + } + parts := bytes.Split(contents, comma) + out = make([]string, len(parts)) + for i, part := range parts { + out[i] = string(part) + } + return +} + +func parseInt(in []byte) (out *big.Int, rest []byte, ok bool) { + contents, rest, ok := parseString(in) + if !ok { + return + } + out = new(big.Int) + + if len(contents) > 0 && contents[0]&0x80 == 0x80 { + // This is a negative number + notBytes := make([]byte, len(contents)) + for i := range notBytes { + notBytes[i] = ^contents[i] + } + out.SetBytes(notBytes) + out.Add(out, bigOne) + out.Neg(out) + } else { + // Positive number + out.SetBytes(contents) + } + ok = true + return +} + +func parseUint32(in []byte) (out uint32, rest []byte, ok bool) { + if len(in) < 4 { + return + } + out = uint32(in[0])<<24 | uint32(in[1])<<16 | uint32(in[2])<<8 | uint32(in[3]) + rest = in[4:] + ok = true + return +} + +const maxPacketSize = 36000 + +func nameListLength(namelist []string) int { + length := 4 /* uint32 length prefix */ + for i, name := range namelist { + if i != 0 { + length++ /* comma */ + } + length += len(name) + } + return length +} + +func intLength(n *big.Int) int { + length := 4 /* length bytes */ + if n.Sign() < 0 { + nMinus1 := new(big.Int).Neg(n) + nMinus1.Sub(nMinus1, bigOne) + bitLen := nMinus1.BitLen() + if bitLen%8 == 0 { + // The number will need 0xff padding + length++ + } + length += (bitLen + 7) / 8 + } else if n.Sign() == 0 { + // A zero is the zero length string + } else { + bitLen := n.BitLen() + if bitLen%8 == 0 { + // The number will need 0x00 padding + length++ + } + length += (bitLen + 7) / 8 + } + + return length +} + +func marshalInt(to []byte, n *big.Int) []byte { + lengthBytes := to + to = to[4:] + length := 0 + + if n.Sign() < 0 { + // A negative number has to be converted to two's-complement + // form. So we'll subtract 1 and invert. If the + // most-significant-bit isn't set then we'll need to pad the + // beginning with 0xff in order to keep the number negative. + nMinus1 := new(big.Int).Neg(n) + nMinus1.Sub(nMinus1, bigOne) + bytes := nMinus1.Bytes() + for i := range bytes { + bytes[i] ^= 0xff + } + if len(bytes) == 0 || bytes[0]&0x80 == 0 { + to[0] = 0xff + to = to[1:] + length++ + } + nBytes := copy(to, bytes) + to = to[nBytes:] + length += nBytes + } else if n.Sign() == 0 { + // A zero is the zero length string + } else { + bytes := n.Bytes() + if len(bytes) > 0 && bytes[0]&0x80 != 0 { + // We'll have to pad this with a 0x00 in order to + // stop it looking like a negative number. + to[0] = 0 + to = to[1:] + length++ + } + nBytes := copy(to, bytes) + to = to[nBytes:] + length += nBytes + } + + lengthBytes[0] = byte(length >> 24) + lengthBytes[1] = byte(length >> 16) + lengthBytes[2] = byte(length >> 8) + lengthBytes[3] = byte(length) + return to +} + +func writeInt(w io.Writer, n *big.Int) { + length := intLength(n) + buf := make([]byte, length) + marshalInt(buf, n) + w.Write(buf) +} + +func writeString(w io.Writer, s []byte) { + var lengthBytes [4]byte + lengthBytes[0] = byte(len(s) >> 24) + lengthBytes[1] = byte(len(s) >> 16) + lengthBytes[2] = byte(len(s) >> 8) + lengthBytes[3] = byte(len(s)) + w.Write(lengthBytes[:]) + w.Write(s) +} + +func stringLength(s []byte) int { + return 4 + len(s) +} + +func marshalString(to []byte, s []byte) []byte { + to[0] = byte(len(s) >> 24) + to[1] = byte(len(s) >> 16) + to[2] = byte(len(s) >> 8) + to[3] = byte(len(s)) + to = to[4:] + copy(to, s) + return to[len(s):] +} + +var bigIntType = reflect.TypeOf((*big.Int)(nil)) + +// Decode a packet into it's corresponding message. +func decode(packet []byte) interface{} { + var msg interface{} + switch packet[0] { + case msgDisconnect: + msg = new(disconnectMsg) + case msgServiceRequest: + msg = new(serviceRequestMsg) + case msgServiceAccept: + msg = new(serviceAcceptMsg) + case msgKexInit: + msg = new(kexInitMsg) + case msgKexDHInit: + msg = new(kexDHInitMsg) + case msgKexDHReply: + msg = new(kexDHReplyMsg) + case msgUserAuthRequest: + msg = new(userAuthRequestMsg) + case msgUserAuthFailure: + msg = new(userAuthFailureMsg) + case msgUserAuthPubKeyOk: + msg = new(userAuthPubKeyOkMsg) + case msgGlobalRequest: + msg = new(globalRequestMsg) + case msgRequestSuccess: + msg = new(channelRequestSuccessMsg) + case msgRequestFailure: + msg = new(channelRequestFailureMsg) + case msgChannelOpen: + msg = new(channelOpenMsg) + case msgChannelOpenConfirm: + msg = new(channelOpenConfirmMsg) + case msgChannelOpenFailure: + msg = new(channelOpenFailureMsg) + case msgChannelWindowAdjust: + msg = new(windowAdjustMsg) + case msgChannelData: + msg = new(channelData) + case msgChannelExtendedData: + msg = new(channelExtendedData) + case msgChannelEOF: + msg = new(channelEOFMsg) + case msgChannelClose: + msg = new(channelCloseMsg) + case msgChannelRequest: + msg = new(channelRequestMsg) + case msgChannelSuccess: + msg = new(channelRequestSuccessMsg) + case msgChannelFailure: + msg = new(channelRequestFailureMsg) + default: + return UnexpectedMessageError{0, packet[0]} + } + if err := unmarshal(msg, packet, packet[0]); err != nil { + return err + } + return msg +} diff --git a/libgo/go/exp/ssh/messages_test.go b/libgo/go/exp/ssh/messages_test.go new file mode 100644 index 0000000..629f3d3 --- /dev/null +++ b/libgo/go/exp/ssh/messages_test.go @@ -0,0 +1,125 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "big" + "rand" + "reflect" + "testing" + "testing/quick" +) + +var intLengthTests = []struct { + val, length int +}{ + {0, 4 + 0}, + {1, 4 + 1}, + {127, 4 + 1}, + {128, 4 + 2}, + {-1, 4 + 1}, +} + +func TestIntLength(t *testing.T) { + for _, test := range intLengthTests { + v := new(big.Int).SetInt64(int64(test.val)) + length := intLength(v) + if length != test.length { + t.Errorf("For %d, got length %d but expected %d", test.val, length, test.length) + } + } +} + +var messageTypes = []interface{}{ + &kexInitMsg{}, + &kexDHInitMsg{}, + &serviceRequestMsg{}, + &serviceAcceptMsg{}, + &userAuthRequestMsg{}, + &channelOpenMsg{}, + &channelOpenConfirmMsg{}, + &channelRequestMsg{}, + &channelRequestSuccessMsg{}, +} + +func TestMarshalUnmarshal(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for i, iface := range messageTypes { + ty := reflect.ValueOf(iface).Type() + + n := 100 + if testing.Short() { + n = 5 + } + for j := 0; j < n; j++ { + v, ok := quick.Value(ty, rand) + if !ok { + t.Errorf("#%d: failed to create value", i) + break + } + + m1 := v.Elem().Interface() + m2 := iface + + marshaled := marshal(msgIgnore, m1) + if err := unmarshal(m2, marshaled, msgIgnore); err != nil { + t.Errorf("#%d failed to unmarshal %#v: %s", i, m1, err) + break + } + + if !reflect.DeepEqual(v.Interface(), m2) { + t.Errorf("#%d\ngot: %#v\nwant:%#v\n%x", i, m2, m1, marshaled) + break + } + } + } +} + +func randomBytes(out []byte, rand *rand.Rand) { + for i := 0; i < len(out); i++ { + out[i] = byte(rand.Int31()) + } +} + +func randomNameList(rand *rand.Rand) []string { + ret := make([]string, rand.Int31()&15) + for i := range ret { + s := make([]byte, 1+(rand.Int31()&15)) + for j := range s { + s[j] = 'a' + uint8(rand.Int31()&15) + } + ret[i] = string(s) + } + return ret +} + +func randomInt(rand *rand.Rand) *big.Int { + return new(big.Int).SetInt64(int64(int32(rand.Uint32()))) +} + +func (*kexInitMsg) Generate(rand *rand.Rand, size int) reflect.Value { + ki := &kexInitMsg{} + randomBytes(ki.Cookie[:], rand) + ki.KexAlgos = randomNameList(rand) + ki.ServerHostKeyAlgos = randomNameList(rand) + ki.CiphersClientServer = randomNameList(rand) + ki.CiphersServerClient = randomNameList(rand) + ki.MACsClientServer = randomNameList(rand) + ki.MACsServerClient = randomNameList(rand) + ki.CompressionClientServer = randomNameList(rand) + ki.CompressionServerClient = randomNameList(rand) + ki.LanguagesClientServer = randomNameList(rand) + ki.LanguagesServerClient = randomNameList(rand) + if rand.Int31()&1 == 1 { + ki.FirstKexFollows = true + } + return reflect.ValueOf(ki) +} + +func (*kexDHInitMsg) Generate(rand *rand.Rand, size int) reflect.Value { + dhi := &kexDHInitMsg{} + dhi.X = randomInt(rand) + return reflect.ValueOf(dhi) +} diff --git a/libgo/go/exp/ssh/server.go b/libgo/go/exp/ssh/server.go new file mode 100644 index 0000000..410cafc --- /dev/null +++ b/libgo/go/exp/ssh/server.go @@ -0,0 +1,645 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "big" + "bytes" + "crypto" + "crypto/rand" + "crypto/rsa" + _ "crypto/sha1" + "crypto/x509" + "encoding/pem" + "net" + "os" + "sync" +) + +// Server represents an SSH server. A Server may have several ServerConnections. +type Server struct { + rsa *rsa.PrivateKey + rsaSerialized []byte + + // NoClientAuth is true if clients are allowed to connect without + // authenticating. + NoClientAuth bool + + // PasswordCallback, if non-nil, is called when a user attempts to + // authenticate using a password. It may be called concurrently from + // several goroutines. + PasswordCallback func(user, password string) bool + + // PubKeyCallback, if non-nil, is called when a client attempts public + // key authentication. It must return true iff the given public key is + // valid for the given user. + PubKeyCallback func(user, algo string, pubkey []byte) bool +} + +// SetRSAPrivateKey sets the private key for a Server. A Server must have a +// private key configured in order to accept connections. The private key must +// be in the form of a PEM encoded, PKCS#1, RSA private key. The file "id_rsa" +// typically contains such a key. +func (s *Server) SetRSAPrivateKey(pemBytes []byte) os.Error { + block, _ := pem.Decode(pemBytes) + if block == nil { + return os.NewError("ssh: no key found") + } + var err os.Error + s.rsa, err = x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return err + } + + s.rsaSerialized = marshalRSA(s.rsa) + return nil +} + +// marshalRSA serializes an RSA private key according to RFC 4256, section 6.6. +func marshalRSA(priv *rsa.PrivateKey) []byte { + e := new(big.Int).SetInt64(int64(priv.E)) + length := stringLength([]byte(hostAlgoRSA)) + length += intLength(e) + length += intLength(priv.N) + + ret := make([]byte, length) + r := marshalString(ret, []byte(hostAlgoRSA)) + r = marshalInt(r, e) + r = marshalInt(r, priv.N) + + return ret +} + +// parseRSA parses an RSA key according to RFC 4256, section 6.6. +func parseRSA(in []byte) (pubKey *rsa.PublicKey, ok bool) { + algo, in, ok := parseString(in) + if !ok || string(algo) != hostAlgoRSA { + return nil, false + } + bigE, in, ok := parseInt(in) + if !ok || bigE.BitLen() > 24 { + return nil, false + } + e := bigE.Int64() + if e < 3 || e&1 == 0 { + return nil, false + } + N, in, ok := parseInt(in) + if !ok || len(in) > 0 { + return nil, false + } + return &rsa.PublicKey{ + N: N, + E: int(e), + }, true +} + +func parseRSASig(in []byte) (sig []byte, ok bool) { + algo, in, ok := parseString(in) + if !ok || string(algo) != hostAlgoRSA { + return nil, false + } + sig, in, ok = parseString(in) + if len(in) > 0 { + ok = false + } + return +} + +// cachedPubKey contains the results of querying whether a public key is +// acceptable for a user. The cache only applies to a single ServerConnection. +type cachedPubKey struct { + user, algo string + pubKey []byte + result bool +} + +const maxCachedPubKeys = 16 + +// ServerConnection represents an incomming connection to a Server. +type ServerConnection struct { + Server *Server + + *transport + + channels map[uint32]*channel + nextChanId uint32 + + // lock protects err and also allows Channels to serialise their writes + // to out. + lock sync.RWMutex + err os.Error + + // cachedPubKeys contains the cache results of tests for public keys. + // Since SSH clients will query whether a public key is acceptable + // before attempting to authenticate with it, we end up with duplicate + // queries for public key validity. + cachedPubKeys []cachedPubKey +} + +// kexDH performs Diffie-Hellman key agreement on a ServerConnection. The +// returned values are given the same names as in RFC 4253, section 8. +func (s *ServerConnection) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *handshakeMagics, hostKeyAlgo string) (H, K []byte, err os.Error) { + packet, err := s.readPacket() + if err != nil { + return + } + var kexDHInit kexDHInitMsg + if err = unmarshal(&kexDHInit, packet, msgKexDHInit); err != nil { + return + } + + if kexDHInit.X.Sign() == 0 || kexDHInit.X.Cmp(group.p) >= 0 { + return nil, nil, os.NewError("client DH parameter out of bounds") + } + + y, err := rand.Int(rand.Reader, group.p) + if err != nil { + return + } + + Y := new(big.Int).Exp(group.g, y, group.p) + kInt := new(big.Int).Exp(kexDHInit.X, y, group.p) + + var serializedHostKey []byte + switch hostKeyAlgo { + case hostAlgoRSA: + serializedHostKey = s.Server.rsaSerialized + default: + return nil, nil, os.NewError("internal error") + } + + h := hashFunc.New() + writeString(h, magics.clientVersion) + writeString(h, magics.serverVersion) + writeString(h, magics.clientKexInit) + writeString(h, magics.serverKexInit) + writeString(h, serializedHostKey) + writeInt(h, kexDHInit.X) + writeInt(h, Y) + K = make([]byte, intLength(kInt)) + marshalInt(K, kInt) + h.Write(K) + + H = h.Sum() + + h.Reset() + h.Write(H) + hh := h.Sum() + + var sig []byte + switch hostKeyAlgo { + case hostAlgoRSA: + sig, err = rsa.SignPKCS1v15(rand.Reader, s.Server.rsa, hashFunc, hh) + if err != nil { + return + } + default: + return nil, nil, os.NewError("internal error") + } + + serializedSig := serializeRSASignature(sig) + + kexDHReply := kexDHReplyMsg{ + HostKey: serializedHostKey, + Y: Y, + Signature: serializedSig, + } + packet = marshal(msgKexDHReply, kexDHReply) + + err = s.writePacket(packet) + return +} + +func serializeRSASignature(sig []byte) []byte { + length := stringLength([]byte(hostAlgoRSA)) + length += stringLength(sig) + + ret := make([]byte, length) + r := marshalString(ret, []byte(hostAlgoRSA)) + r = marshalString(r, sig) + + return ret +} + +// serverVersion is the fixed identification string that Server will use. +var serverVersion = []byte("SSH-2.0-Go\r\n") + +// buildDataSignedForAuth returns the data that is signed in order to prove +// posession of a private key. See RFC 4252, section 7. +func buildDataSignedForAuth(sessionId []byte, req userAuthRequestMsg, algo, pubKey []byte) []byte { + user := []byte(req.User) + service := []byte(req.Service) + method := []byte(req.Method) + + length := stringLength(sessionId) + length += 1 + length += stringLength(user) + length += stringLength(service) + length += stringLength(method) + length += 1 + length += stringLength(algo) + length += stringLength(pubKey) + + ret := make([]byte, length) + r := marshalString(ret, sessionId) + r[0] = msgUserAuthRequest + r = r[1:] + r = marshalString(r, user) + r = marshalString(r, service) + r = marshalString(r, method) + r[0] = 1 + r = r[1:] + r = marshalString(r, algo) + r = marshalString(r, pubKey) + return ret +} + +// Handshake performs an SSH transport and client authentication on the given ServerConnection. +func (s *ServerConnection) Handshake(conn net.Conn) os.Error { + var magics handshakeMagics + s.transport = newTransport(conn, rand.Reader) + + if _, err := conn.Write(serverVersion); err != nil { + return err + } + magics.serverVersion = serverVersion[:len(serverVersion)-2] + + version, ok := readVersion(s.transport) + if !ok { + return os.NewError("failed to read version string from client") + } + magics.clientVersion = version + + serverKexInit := kexInitMsg{ + KexAlgos: supportedKexAlgos, + ServerHostKeyAlgos: supportedHostKeyAlgos, + CiphersClientServer: supportedCiphers, + CiphersServerClient: supportedCiphers, + MACsClientServer: supportedMACs, + MACsServerClient: supportedMACs, + CompressionClientServer: supportedCompressions, + CompressionServerClient: supportedCompressions, + } + kexInitPacket := marshal(msgKexInit, serverKexInit) + magics.serverKexInit = kexInitPacket + + if err := s.writePacket(kexInitPacket); err != nil { + return err + } + + packet, err := s.readPacket() + if err != nil { + return err + } + + magics.clientKexInit = packet + + var clientKexInit kexInitMsg + if err = unmarshal(&clientKexInit, packet, msgKexInit); err != nil { + return err + } + + kexAlgo, hostKeyAlgo, ok := findAgreedAlgorithms(s.transport, &clientKexInit, &serverKexInit) + if !ok { + return os.NewError("ssh: no common algorithms") + } + + if clientKexInit.FirstKexFollows && kexAlgo != clientKexInit.KexAlgos[0] { + // The client sent a Kex message for the wrong algorithm, + // which we have to ignore. + _, err := s.readPacket() + if err != nil { + return err + } + } + + var H, K []byte + var hashFunc crypto.Hash + switch kexAlgo { + case kexAlgoDH14SHA1: + hashFunc = crypto.SHA1 + dhGroup14Once.Do(initDHGroup14) + H, K, err = s.kexDH(dhGroup14, hashFunc, &magics, hostKeyAlgo) + default: + err = os.NewError("ssh: internal error") + } + + if err != nil { + return err + } + + packet = []byte{msgNewKeys} + if err = s.writePacket(packet); err != nil { + return err + } + if err = s.transport.writer.setupKeys(serverKeys, K, H, H, hashFunc); err != nil { + return err + } + + if packet, err = s.readPacket(); err != nil { + return err + } + if packet[0] != msgNewKeys { + return UnexpectedMessageError{msgNewKeys, packet[0]} + } + + s.transport.reader.setupKeys(clientKeys, K, H, H, hashFunc) + + packet, err = s.readPacket() + if err != nil { + return err + } + + var serviceRequest serviceRequestMsg + if err = unmarshal(&serviceRequest, packet, msgServiceRequest); err != nil { + return err + } + if serviceRequest.Service != serviceUserAuth { + return os.NewError("ssh: requested service '" + serviceRequest.Service + "' before authenticating") + } + + serviceAccept := serviceAcceptMsg{ + Service: serviceUserAuth, + } + packet = marshal(msgServiceAccept, serviceAccept) + if err = s.writePacket(packet); err != nil { + return err + } + + if err = s.authenticate(H); err != nil { + return err + } + + s.channels = make(map[uint32]*channel) + return nil +} + +func isAcceptableAlgo(algo string) bool { + return algo == hostAlgoRSA +} + +// testPubKey returns true if the given public key is acceptable for the user. +func (s *ServerConnection) testPubKey(user, algo string, pubKey []byte) bool { + if s.Server.PubKeyCallback == nil || !isAcceptableAlgo(algo) { + return false + } + + for _, c := range s.cachedPubKeys { + if c.user == user && c.algo == algo && bytes.Equal(c.pubKey, pubKey) { + return c.result + } + } + + result := s.Server.PubKeyCallback(user, algo, pubKey) + if len(s.cachedPubKeys) < maxCachedPubKeys { + c := cachedPubKey{ + user: user, + algo: algo, + pubKey: make([]byte, len(pubKey)), + result: result, + } + copy(c.pubKey, pubKey) + s.cachedPubKeys = append(s.cachedPubKeys, c) + } + + return result +} + +func (s *ServerConnection) authenticate(H []byte) os.Error { + var userAuthReq userAuthRequestMsg + var err os.Error + var packet []byte + +userAuthLoop: + for { + if packet, err = s.readPacket(); err != nil { + return err + } + if err = unmarshal(&userAuthReq, packet, msgUserAuthRequest); err != nil { + return err + } + + if userAuthReq.Service != serviceSSH { + return os.NewError("ssh: client attempted to negotiate for unknown service: " + userAuthReq.Service) + } + + switch userAuthReq.Method { + case "none": + if s.Server.NoClientAuth { + break userAuthLoop + } + case "password": + if s.Server.PasswordCallback == nil { + break + } + payload := userAuthReq.Payload + if len(payload) < 1 || payload[0] != 0 { + return ParseError{msgUserAuthRequest} + } + payload = payload[1:] + password, payload, ok := parseString(payload) + if !ok || len(payload) > 0 { + return ParseError{msgUserAuthRequest} + } + + if s.Server.PasswordCallback(userAuthReq.User, string(password)) { + break userAuthLoop + } + case "publickey": + if s.Server.PubKeyCallback == nil { + break + } + payload := userAuthReq.Payload + if len(payload) < 1 { + return ParseError{msgUserAuthRequest} + } + isQuery := payload[0] == 0 + payload = payload[1:] + algoBytes, payload, ok := parseString(payload) + if !ok { + return ParseError{msgUserAuthRequest} + } + algo := string(algoBytes) + + pubKey, payload, ok := parseString(payload) + if !ok { + return ParseError{msgUserAuthRequest} + } + if isQuery { + // The client can query if the given public key + // would be ok. + if len(payload) > 0 { + return ParseError{msgUserAuthRequest} + } + if s.testPubKey(userAuthReq.User, algo, pubKey) { + okMsg := userAuthPubKeyOkMsg{ + Algo: algo, + PubKey: string(pubKey), + } + if err = s.writePacket(marshal(msgUserAuthPubKeyOk, okMsg)); err != nil { + return err + } + continue userAuthLoop + } + } else { + sig, payload, ok := parseString(payload) + if !ok || len(payload) > 0 { + return ParseError{msgUserAuthRequest} + } + if !isAcceptableAlgo(algo) { + break + } + rsaSig, ok := parseRSASig(sig) + if !ok { + return ParseError{msgUserAuthRequest} + } + signedData := buildDataSignedForAuth(H, userAuthReq, algoBytes, pubKey) + switch algo { + case hostAlgoRSA: + hashFunc := crypto.SHA1 + h := hashFunc.New() + h.Write(signedData) + digest := h.Sum() + rsaKey, ok := parseRSA(pubKey) + if !ok { + return ParseError{msgUserAuthRequest} + } + if rsa.VerifyPKCS1v15(rsaKey, hashFunc, digest, rsaSig) != nil { + return ParseError{msgUserAuthRequest} + } + default: + return os.NewError("ssh: isAcceptableAlgo incorrect") + } + if s.testPubKey(userAuthReq.User, algo, pubKey) { + break userAuthLoop + } + } + } + + var failureMsg userAuthFailureMsg + if s.Server.PasswordCallback != nil { + failureMsg.Methods = append(failureMsg.Methods, "password") + } + if s.Server.PubKeyCallback != nil { + failureMsg.Methods = append(failureMsg.Methods, "publickey") + } + + if len(failureMsg.Methods) == 0 { + return os.NewError("ssh: no authentication methods configured but NoClientAuth is also false") + } + + if err = s.writePacket(marshal(msgUserAuthFailure, failureMsg)); err != nil { + return err + } + } + + packet = []byte{msgUserAuthSuccess} + if err = s.writePacket(packet); err != nil { + return err + } + + return nil +} + +const defaultWindowSize = 32768 + +// Accept reads and processes messages on a ServerConnection. It must be called +// in order to demultiplex messages to any resulting Channels. +func (s *ServerConnection) Accept() (Channel, os.Error) { + if s.err != nil { + return nil, s.err + } + + for { + packet, err := s.readPacket() + if err != nil { + + s.lock.Lock() + s.err = err + s.lock.Unlock() + + for _, c := range s.channels { + c.dead = true + c.handleData(nil) + } + + return nil, err + } + + switch msg := decode(packet).(type) { + case *channelOpenMsg: + c := new(channel) + c.chanType = msg.ChanType + c.theirId = msg.PeersId + c.theirWindow = msg.PeersWindow + c.maxPacketSize = msg.MaxPacketSize + c.extraData = msg.TypeSpecificData + c.myWindow = defaultWindowSize + c.serverConn = s + c.cond = sync.NewCond(&c.lock) + c.pendingData = make([]byte, c.myWindow) + + s.lock.Lock() + c.myId = s.nextChanId + s.nextChanId++ + s.channels[c.myId] = c + s.lock.Unlock() + return c, nil + + case *channelRequestMsg: + s.lock.Lock() + c, ok := s.channels[msg.PeersId] + if !ok { + continue + } + c.handlePacket(msg) + s.lock.Unlock() + + case *channelData: + s.lock.Lock() + c, ok := s.channels[msg.PeersId] + if !ok { + continue + } + c.handleData(msg.Payload) + s.lock.Unlock() + + case *channelEOFMsg: + s.lock.Lock() + c, ok := s.channels[msg.PeersId] + if !ok { + continue + } + c.handlePacket(msg) + s.lock.Unlock() + + case *channelCloseMsg: + s.lock.Lock() + c, ok := s.channels[msg.PeersId] + if !ok { + continue + } + c.handlePacket(msg) + s.lock.Unlock() + + case *globalRequestMsg: + if msg.WantReply { + if err := s.writePacket([]byte{msgRequestFailure}); err != nil { + return nil, err + } + } + + case UnexpectedMessageError: + return nil, msg + case *disconnectMsg: + return nil, os.EOF + default: + // Unknown message. Ignore. + } + } + + panic("unreachable") +} diff --git a/libgo/go/exp/ssh/server_shell.go b/libgo/go/exp/ssh/server_shell.go new file mode 100644 index 0000000..0e9967a --- /dev/null +++ b/libgo/go/exp/ssh/server_shell.go @@ -0,0 +1,400 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "os" +) + +// ServerShell contains the state for running a VT100 terminal that is capable +// of reading lines of input. +type ServerShell struct { + c Channel + prompt string + + // line is the current line being entered. + line []byte + // pos is the logical position of the cursor in line + pos int + + // cursorX contains the current X value of the cursor where the left + // edge is 0. cursorY contains the row number where the first row of + // the current line is 0. + cursorX, cursorY int + // maxLine is the greatest value of cursorY so far. + maxLine int + + termWidth, termHeight int + + // outBuf contains the terminal data to be sent. + outBuf []byte + // remainder contains the remainder of any partial key sequences after + // a read. It aliases into inBuf. + remainder []byte + inBuf [256]byte +} + +// NewServerShell runs a VT100 terminal on the given channel. prompt is a +// string that is written at the start of each input line. For example: "> ". +func NewServerShell(c Channel, prompt string) *ServerShell { + return &ServerShell{ + c: c, + prompt: prompt, + termWidth: 80, + termHeight: 24, + } +} + +const ( + keyCtrlD = 4 + keyEnter = '\r' + keyEscape = 27 + keyBackspace = 127 + keyUnknown = 256 + iota + keyUp + keyDown + keyLeft + keyRight + keyAltLeft + keyAltRight +) + +// bytesToKey tries to parse a key sequence from b. If successful, it returns +// the key and the remainder of the input. Otherwise it returns -1. +func bytesToKey(b []byte) (int, []byte) { + if len(b) == 0 { + return -1, nil + } + + if b[0] != keyEscape { + return int(b[0]), b[1:] + } + + if len(b) >= 3 && b[0] == keyEscape && b[1] == '[' { + switch b[2] { + case 'A': + return keyUp, b[3:] + case 'B': + return keyDown, b[3:] + case 'C': + return keyRight, b[3:] + case 'D': + return keyLeft, b[3:] + } + } + + if len(b) >= 6 && b[0] == keyEscape && b[1] == '[' && b[2] == '1' && b[3] == ';' && b[4] == '3' { + switch b[5] { + case 'C': + return keyAltRight, b[6:] + case 'D': + return keyAltLeft, b[6:] + } + } + + // If we get here then we have a key that we don't recognise, or a + // partial sequence. It's not clear how one should find the end of a + // sequence without knowing them all, but it seems that [a-zA-Z] only + // appears at the end of a sequence. + for i, c := range b[0:] { + if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' { + return keyUnknown, b[i+1:] + } + } + + return -1, b +} + +// queue appends data to the end of ss.outBuf +func (ss *ServerShell) queue(data []byte) { + if len(ss.outBuf)+len(data) > cap(ss.outBuf) { + newOutBuf := make([]byte, len(ss.outBuf), 2*(len(ss.outBuf)+len(data))) + copy(newOutBuf, ss.outBuf) + ss.outBuf = newOutBuf + } + + oldLen := len(ss.outBuf) + ss.outBuf = ss.outBuf[:len(ss.outBuf)+len(data)] + copy(ss.outBuf[oldLen:], data) +} + +var eraseUnderCursor = []byte{' ', keyEscape, '[', 'D'} + +func isPrintable(key int) bool { + return key >= 32 && key < 127 +} + +// moveCursorToPos appends data to ss.outBuf which will move the cursor to the +// given, logical position in the text. +func (ss *ServerShell) moveCursorToPos(pos int) { + x := len(ss.prompt) + pos + y := x / ss.termWidth + x = x % ss.termWidth + + up := 0 + if y < ss.cursorY { + up = ss.cursorY - y + } + + down := 0 + if y > ss.cursorY { + down = y - ss.cursorY + } + + left := 0 + if x < ss.cursorX { + left = ss.cursorX - x + } + + right := 0 + if x > ss.cursorX { + right = x - ss.cursorX + } + + movement := make([]byte, 3*(up+down+left+right)) + m := movement + for i := 0; i < up; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'A' + m = m[3:] + } + for i := 0; i < down; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'B' + m = m[3:] + } + for i := 0; i < left; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'D' + m = m[3:] + } + for i := 0; i < right; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'C' + m = m[3:] + } + + ss.cursorX = x + ss.cursorY = y + ss.queue(movement) +} + +const maxLineLength = 4096 + +// handleKey processes the given key and, optionally, returns a line of text +// that the user has entered. +func (ss *ServerShell) handleKey(key int) (line string, ok bool) { + switch key { + case keyBackspace: + if ss.pos == 0 { + return + } + ss.pos-- + + copy(ss.line[ss.pos:], ss.line[1+ss.pos:]) + ss.line = ss.line[:len(ss.line)-1] + ss.writeLine(ss.line[ss.pos:]) + ss.moveCursorToPos(ss.pos) + ss.queue(eraseUnderCursor) + case keyAltLeft: + // move left by a word. + if ss.pos == 0 { + return + } + ss.pos-- + for ss.pos > 0 { + if ss.line[ss.pos] != ' ' { + break + } + ss.pos-- + } + for ss.pos > 0 { + if ss.line[ss.pos] == ' ' { + ss.pos++ + break + } + ss.pos-- + } + ss.moveCursorToPos(ss.pos) + case keyAltRight: + // move right by a word. + for ss.pos < len(ss.line) { + if ss.line[ss.pos] == ' ' { + break + } + ss.pos++ + } + for ss.pos < len(ss.line) { + if ss.line[ss.pos] != ' ' { + break + } + ss.pos++ + } + ss.moveCursorToPos(ss.pos) + case keyLeft: + if ss.pos == 0 { + return + } + ss.pos-- + ss.moveCursorToPos(ss.pos) + case keyRight: + if ss.pos == len(ss.line) { + return + } + ss.pos++ + ss.moveCursorToPos(ss.pos) + case keyEnter: + ss.moveCursorToPos(len(ss.line)) + ss.queue([]byte("\r\n")) + line = string(ss.line) + ok = true + ss.line = ss.line[:0] + ss.pos = 0 + ss.cursorX = 0 + ss.cursorY = 0 + ss.maxLine = 0 + default: + if !isPrintable(key) { + return + } + if len(ss.line) == maxLineLength { + return + } + if len(ss.line) == cap(ss.line) { + newLine := make([]byte, len(ss.line), 2*(1+len(ss.line))) + copy(newLine, ss.line) + ss.line = newLine + } + ss.line = ss.line[:len(ss.line)+1] + copy(ss.line[ss.pos+1:], ss.line[ss.pos:]) + ss.line[ss.pos] = byte(key) + ss.writeLine(ss.line[ss.pos:]) + ss.pos++ + ss.moveCursorToPos(ss.pos) + } + return +} + +func (ss *ServerShell) writeLine(line []byte) { + for len(line) != 0 { + if ss.cursorX == ss.termWidth { + ss.queue([]byte("\r\n")) + ss.cursorX = 0 + ss.cursorY++ + if ss.cursorY > ss.maxLine { + ss.maxLine = ss.cursorY + } + } + + remainingOnLine := ss.termWidth - ss.cursorX + todo := len(line) + if todo > remainingOnLine { + todo = remainingOnLine + } + ss.queue(line[:todo]) + ss.cursorX += todo + line = line[todo:] + } +} + +// parsePtyRequest parses the payload of the pty-req message and extracts the +// dimensions of the terminal. See RFC 4254, section 6.2. +func parsePtyRequest(s []byte) (width, height int, ok bool) { + _, s, ok = parseString(s) + if !ok { + return + } + width32, s, ok := parseUint32(s) + if !ok { + return + } + height32, _, ok := parseUint32(s) + width = int(width32) + height = int(height32) + if width < 1 { + ok = false + } + if height < 1 { + ok = false + } + return +} + +func (ss *ServerShell) Write(buf []byte) (n int, err os.Error) { + return ss.c.Write(buf) +} + +// ReadLine returns a line of input from the terminal. +func (ss *ServerShell) ReadLine() (line string, err os.Error) { + ss.writeLine([]byte(ss.prompt)) + ss.c.Write(ss.outBuf) + ss.outBuf = ss.outBuf[:0] + + for { + // ss.remainder is a slice at the beginning of ss.inBuf + // containing a partial key sequence + readBuf := ss.inBuf[len(ss.remainder):] + var n int + n, err = ss.c.Read(readBuf) + if err == nil { + ss.remainder = ss.inBuf[:n+len(ss.remainder)] + rest := ss.remainder + lineOk := false + for !lineOk { + var key int + key, rest = bytesToKey(rest) + if key < 0 { + break + } + if key == keyCtrlD { + return "", os.EOF + } + line, lineOk = ss.handleKey(key) + } + if len(rest) > 0 { + n := copy(ss.inBuf[:], rest) + ss.remainder = ss.inBuf[:n] + } else { + ss.remainder = nil + } + ss.c.Write(ss.outBuf) + ss.outBuf = ss.outBuf[:0] + if lineOk { + return + } + continue + } + + if req, ok := err.(ChannelRequest); ok { + ok := false + switch req.Request { + case "pty-req": + ss.termWidth, ss.termHeight, ok = parsePtyRequest(req.Payload) + if !ok { + ss.termWidth = 80 + ss.termHeight = 24 + } + case "shell": + ok = true + if len(req.Payload) > 0 { + // We don't accept any commands, only the default shell. + ok = false + } + case "env": + ok = true + } + if req.WantReply { + ss.c.AckRequest(ok) + } + } else { + return "", err + } + } + panic("unreachable") +} diff --git a/libgo/go/exp/ssh/server_shell_test.go b/libgo/go/exp/ssh/server_shell_test.go new file mode 100644 index 0000000..622cf7c --- /dev/null +++ b/libgo/go/exp/ssh/server_shell_test.go @@ -0,0 +1,134 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "testing" + "os" +) + +type MockChannel struct { + toSend []byte + bytesPerRead int + received []byte +} + +func (c *MockChannel) Accept() os.Error { + return nil +} + +func (c *MockChannel) Reject(RejectionReason, string) os.Error { + return nil +} + +func (c *MockChannel) Read(data []byte) (n int, err os.Error) { + n = len(data) + if n == 0 { + return + } + if n > len(c.toSend) { + n = len(c.toSend) + } + if n == 0 { + return 0, os.EOF + } + if c.bytesPerRead > 0 && n > c.bytesPerRead { + n = c.bytesPerRead + } + copy(data, c.toSend[:n]) + c.toSend = c.toSend[n:] + return +} + +func (c *MockChannel) Write(data []byte) (n int, err os.Error) { + c.received = append(c.received, data...) + return len(data), nil +} + +func (c *MockChannel) Close() os.Error { + return nil +} + +func (c *MockChannel) AckRequest(ok bool) os.Error { + return nil +} + +func (c *MockChannel) ChannelType() string { + return "" +} + +func (c *MockChannel) ExtraData() []byte { + return nil +} + +func TestClose(t *testing.T) { + c := &MockChannel{} + ss := NewServerShell(c, "> ") + line, err := ss.ReadLine() + if line != "" { + t.Errorf("Expected empty line but got: %s", line) + } + if err != os.EOF { + t.Errorf("Error should have been EOF but got: %s", err) + } +} + +var keyPressTests = []struct { + in string + line string + err os.Error +}{ + { + "", + "", + os.EOF, + }, + { + "\r", + "", + nil, + }, + { + "foo\r", + "foo", + nil, + }, + { + "a\x1b[Cb\r", // right + "ab", + nil, + }, + { + "a\x1b[Db\r", // left + "ba", + nil, + }, + { + "a\177b\r", // backspace + "b", + nil, + }, +} + +func TestKeyPresses(t *testing.T) { + for i, test := range keyPressTests { + for j := 0; j < len(test.in); j++ { + c := &MockChannel{ + toSend: []byte(test.in), + bytesPerRead: j, + } + ss := NewServerShell(c, "> ") + line, err := ss.ReadLine() + if line != test.line { + t.Errorf("Line resulting from test %d (%d bytes per read) was '%s', expected '%s'", i, j, line, test.line) + break + } + if err != test.err { + t.Errorf("Error resulting from test %d (%d bytes per read) was '%v', expected '%v'", i, j, err, test.err) + break + } + } + } +} diff --git a/libgo/go/exp/ssh/transport.go b/libgo/go/exp/ssh/transport.go new file mode 100644 index 0000000..5994004 --- /dev/null +++ b/libgo/go/exp/ssh/transport.go @@ -0,0 +1,369 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "bufio" + "crypto" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/subtle" + "hash" + "io" + "net" + "os" + "sync" +) + +const ( + paddingMultiple = 16 // TODO(dfc) does this need to be configurable? +) + +// filteredConn reduces the set of methods exposed when embeddeding +// a net.Conn inside ssh.transport. +// TODO(dfc) suggestions for a better name will be warmly received. +type filteredConn interface { + // Close closes the connection. + Close() os.Error + + // LocalAddr returns the local network address. + LocalAddr() net.Addr + + // RemoteAddr returns the remote network address. + RemoteAddr() net.Addr +} + +// Types implementing packetWriter provide the ability to send packets to +// an SSH peer. +type packetWriter interface { + // Encrypt and send a packet of data to the remote peer. + writePacket(packet []byte) os.Error +} + +// transport represents the SSH connection to the remote peer. +type transport struct { + reader + writer + + filteredConn +} + +// reader represents the incoming connection state. +type reader struct { + io.Reader + common +} + +// writer represnts the outgoing connection state. +type writer struct { + *sync.Mutex // protects writer.Writer from concurrent writes + *bufio.Writer + paddingMultiple int + rand io.Reader + common +} + +// common represents the cipher state needed to process messages in a single +// direction. +type common struct { + seqNum uint32 + mac hash.Hash + cipher cipher.Stream + + cipherAlgo string + macAlgo string + compressionAlgo string +} + +// Read and decrypt a single packet from the remote peer. +func (r *reader) readOnePacket() ([]byte, os.Error) { + var lengthBytes = make([]byte, 5) + var macSize uint32 + + if _, err := io.ReadFull(r, lengthBytes); err != nil { + return nil, err + } + + if r.cipher != nil { + r.cipher.XORKeyStream(lengthBytes, lengthBytes) + } + + if r.mac != nil { + r.mac.Reset() + seqNumBytes := []byte{ + byte(r.seqNum >> 24), + byte(r.seqNum >> 16), + byte(r.seqNum >> 8), + byte(r.seqNum), + } + r.mac.Write(seqNumBytes) + r.mac.Write(lengthBytes) + macSize = uint32(r.mac.Size()) + } + + length := uint32(lengthBytes[0])<<24 | uint32(lengthBytes[1])<<16 | uint32(lengthBytes[2])<<8 | uint32(lengthBytes[3]) + paddingLength := uint32(lengthBytes[4]) + + if length <= paddingLength+1 { + return nil, os.NewError("invalid packet length") + } + if length > maxPacketSize { + return nil, os.NewError("packet too large") + } + + packet := make([]byte, length-1+macSize) + if _, err := io.ReadFull(r, packet); err != nil { + return nil, err + } + mac := packet[length-1:] + if r.cipher != nil { + r.cipher.XORKeyStream(packet, packet[:length-1]) + } + + if r.mac != nil { + r.mac.Write(packet[:length-1]) + if subtle.ConstantTimeCompare(r.mac.Sum(), mac) != 1 { + return nil, os.NewError("ssh: MAC failure") + } + } + + r.seqNum++ + return packet[:length-paddingLength-1], nil +} + +// Read and decrypt next packet discarding debug and noop messages. +func (t *transport) readPacket() ([]byte, os.Error) { + for { + packet, err := t.readOnePacket() + if err != nil { + return nil, err + } + if packet[0] != msgIgnore && packet[0] != msgDebug { + return packet, nil + } + } + panic("unreachable") +} + +// Encrypt and send a packet of data to the remote peer. +func (w *writer) writePacket(packet []byte) os.Error { + w.Mutex.Lock() + defer w.Mutex.Unlock() + + paddingLength := paddingMultiple - (5+len(packet))%paddingMultiple + if paddingLength < 4 { + paddingLength += paddingMultiple + } + + length := len(packet) + 1 + paddingLength + lengthBytes := []byte{ + byte(length >> 24), + byte(length >> 16), + byte(length >> 8), + byte(length), + byte(paddingLength), + } + padding := make([]byte, paddingLength) + _, err := io.ReadFull(w.rand, padding) + if err != nil { + return err + } + + if w.mac != nil { + w.mac.Reset() + seqNumBytes := []byte{ + byte(w.seqNum >> 24), + byte(w.seqNum >> 16), + byte(w.seqNum >> 8), + byte(w.seqNum), + } + w.mac.Write(seqNumBytes) + w.mac.Write(lengthBytes) + w.mac.Write(packet) + w.mac.Write(padding) + } + + // TODO(dfc) lengthBytes, packet and padding should be + // subslices of a single buffer + if w.cipher != nil { + w.cipher.XORKeyStream(lengthBytes, lengthBytes) + w.cipher.XORKeyStream(packet, packet) + w.cipher.XORKeyStream(padding, padding) + } + + if _, err := w.Write(lengthBytes); err != nil { + return err + } + if _, err := w.Write(packet); err != nil { + return err + } + if _, err := w.Write(padding); err != nil { + return err + } + + if w.mac != nil { + if _, err := w.Write(w.mac.Sum()); err != nil { + return err + } + } + + if err := w.Flush(); err != nil { + return err + } + w.seqNum++ + return err +} + +// Send a message to the remote peer +func (t *transport) sendMessage(typ uint8, msg interface{}) os.Error { + packet := marshal(typ, msg) + return t.writePacket(packet) +} + +func newTransport(conn net.Conn, rand io.Reader) *transport { + return &transport{ + reader: reader{ + Reader: bufio.NewReader(conn), + }, + writer: writer{ + Writer: bufio.NewWriter(conn), + rand: rand, + Mutex: new(sync.Mutex), + }, + filteredConn: conn, + } +} + +type direction struct { + ivTag []byte + keyTag []byte + macKeyTag []byte +} + +// TODO(dfc) can this be made a constant ? +var ( + serverKeys = direction{[]byte{'B'}, []byte{'D'}, []byte{'F'}} + clientKeys = direction{[]byte{'A'}, []byte{'C'}, []byte{'E'}} +) + +// setupKeys sets the cipher and MAC keys from K, H and sessionId, as +// described in RFC 4253, section 6.4. direction should either be serverKeys +// (to setup server->client keys) or clientKeys (for client->server keys). +func (c *common) setupKeys(d direction, K, H, sessionId []byte, hashFunc crypto.Hash) os.Error { + h := hashFunc.New() + + blockSize := 16 + keySize := 16 + macKeySize := 20 + + iv := make([]byte, blockSize) + key := make([]byte, keySize) + macKey := make([]byte, macKeySize) + generateKeyMaterial(iv, d.ivTag, K, H, sessionId, h) + generateKeyMaterial(key, d.keyTag, K, H, sessionId, h) + generateKeyMaterial(macKey, d.macKeyTag, K, H, sessionId, h) + + c.mac = truncatingMAC{12, hmac.NewSHA1(macKey)} + aes, err := aes.NewCipher(key) + if err != nil { + return err + } + c.cipher = cipher.NewCTR(aes, iv) + return nil +} + +// generateKeyMaterial fills out with key material generated from tag, K, H +// and sessionId, as specified in RFC 4253, section 7.2. +func generateKeyMaterial(out, tag []byte, K, H, sessionId []byte, h hash.Hash) { + var digestsSoFar []byte + + for len(out) > 0 { + h.Reset() + h.Write(K) + h.Write(H) + + if len(digestsSoFar) == 0 { + h.Write(tag) + h.Write(sessionId) + } else { + h.Write(digestsSoFar) + } + + digest := h.Sum() + n := copy(out, digest) + out = out[n:] + if len(out) > 0 { + digestsSoFar = append(digestsSoFar, digest...) + } + } +} + +// truncatingMAC wraps around a hash.Hash and truncates the output digest to +// a given size. +type truncatingMAC struct { + length int + hmac hash.Hash +} + +func (t truncatingMAC) Write(data []byte) (int, os.Error) { + return t.hmac.Write(data) +} + +func (t truncatingMAC) Sum() []byte { + digest := t.hmac.Sum() + return digest[:t.length] +} + +func (t truncatingMAC) Reset() { + t.hmac.Reset() +} + +func (t truncatingMAC) Size() int { + return t.length +} + +// maxVersionStringBytes is the maximum number of bytes that we'll accept as a +// version string. In the event that the client is talking a different protocol +// we need to set a limit otherwise we will keep using more and more memory +// while searching for the end of the version handshake. +const maxVersionStringBytes = 1024 + +// Read version string as specified by RFC 4253, section 4.2. +func readVersion(r io.Reader) (versionString []byte, ok bool) { + versionString = make([]byte, 0, 64) + seenCR := false + + var buf [1]byte +forEachByte: + for len(versionString) < maxVersionStringBytes { + _, err := io.ReadFull(r, buf[:]) + if err != nil { + return + } + b := buf[0] + + if !seenCR { + if b == '\r' { + seenCR = true + } + } else { + if b == '\n' { + ok = true + break forEachByte + } else { + seenCR = false + } + } + versionString = append(versionString, b) + } + + if ok { + // We need to remove the CR from versionString + versionString = versionString[:len(versionString)-1] + } + + return +} diff --git a/libgo/go/exp/ssh/transport_test.go b/libgo/go/exp/ssh/transport_test.go new file mode 100644 index 0000000..9a610a7 --- /dev/null +++ b/libgo/go/exp/ssh/transport_test.go @@ -0,0 +1,37 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "bufio" + "bytes" + "testing" +) + +func TestReadVersion(t *testing.T) { + buf := []byte(serverVersion) + result, ok := readVersion(bufio.NewReader(bytes.NewBuffer(buf))) + if !ok { + t.Error("readVersion didn't read version correctly") + } + if !bytes.Equal(buf[:len(buf)-2], result) { + t.Error("version read did not match expected") + } +} + +func TestReadVersionTooLong(t *testing.T) { + buf := make([]byte, maxVersionStringBytes+1) + if _, ok := readVersion(bufio.NewReader(bytes.NewBuffer(buf))); ok { + t.Errorf("readVersion consumed %d bytes without error", len(buf)) + } +} + +func TestReadVersionWithoutCRLF(t *testing.T) { + buf := []byte(serverVersion) + buf = buf[:len(buf)-1] + if _, ok := readVersion(bufio.NewReader(bytes.NewBuffer(buf))); ok { + t.Error("readVersion did not notice \\n was missing") + } +} diff --git a/libgo/go/exp/template/html/attr.go b/libgo/go/exp/template/html/attr.go new file mode 100644 index 0000000..6a36c7b --- /dev/null +++ b/libgo/go/exp/template/html/attr.go @@ -0,0 +1,175 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +// attrTypeMap[n] describes the value of the given attribute. +// If an attribute affects (or can mask) the encoding or interpretation of +// other content, or affects the contents, idempotency, or credentials of a +// network message, then the value in this map is contentTypeUnsafe. +// This map is derived from HTML5, specifically +// http://www.w3.org/TR/html5/Overview.html#attributes-1 +// as well as "%URI"-typed attributes from +// http://www.w3.org/TR/html4/index/attributes.html +var attrTypeMap = map[string]contentType{ + "accept": contentTypePlain, + "accept-charset": contentTypeUnsafe, + "action": contentTypeURL, + "alt": contentTypePlain, + "archive": contentTypeURL, + "async": contentTypeUnsafe, + "autocomplete": contentTypePlain, + "autofocus": contentTypePlain, + "autoplay": contentTypePlain, + "background": contentTypeURL, + "border": contentTypePlain, + "checked": contentTypePlain, + "cite": contentTypeURL, + "challenge": contentTypeUnsafe, + "charset": contentTypeUnsafe, + "class": contentTypePlain, + "classid": contentTypeURL, + "codebase": contentTypeURL, + "cols": contentTypePlain, + "colspan": contentTypePlain, + "content": contentTypeUnsafe, + "contenteditable": contentTypePlain, + "contextmenu": contentTypePlain, + "controls": contentTypePlain, + "coords": contentTypePlain, + "crossorigin": contentTypeUnsafe, + "data": contentTypeURL, + "datetime": contentTypePlain, + "default": contentTypePlain, + "defer": contentTypeUnsafe, + "dir": contentTypePlain, + "dirname": contentTypePlain, + "disabled": contentTypePlain, + "draggable": contentTypePlain, + "dropzone": contentTypePlain, + "enctype": contentTypeUnsafe, + "for": contentTypePlain, + "form": contentTypeUnsafe, + "formaction": contentTypeURL, + "formenctype": contentTypeUnsafe, + "formmethod": contentTypeUnsafe, + "formnovalidate": contentTypeUnsafe, + "formtarget": contentTypePlain, + "headers": contentTypePlain, + "height": contentTypePlain, + "hidden": contentTypePlain, + "high": contentTypePlain, + "href": contentTypeURL, + "hreflang": contentTypePlain, + "http-equiv": contentTypeUnsafe, + "icon": contentTypeURL, + "id": contentTypePlain, + "ismap": contentTypePlain, + "keytype": contentTypeUnsafe, + "kind": contentTypePlain, + "label": contentTypePlain, + "lang": contentTypePlain, + "language": contentTypeUnsafe, + "list": contentTypePlain, + "longdesc": contentTypeURL, + "loop": contentTypePlain, + "low": contentTypePlain, + "manifest": contentTypeURL, + "max": contentTypePlain, + "maxlength": contentTypePlain, + "media": contentTypePlain, + "mediagroup": contentTypePlain, + "method": contentTypeUnsafe, + "min": contentTypePlain, + "multiple": contentTypePlain, + "name": contentTypePlain, + "novalidate": contentTypeUnsafe, + // Skip handler names from + // http://www.w3.org/TR/html5/Overview.html#event-handlers-on-elements-document-objects-and-window-objects + // since we have special handling in attrType. + "open": contentTypePlain, + "optimum": contentTypePlain, + "pattern": contentTypeUnsafe, + "placeholder": contentTypePlain, + "poster": contentTypeURL, + "profile": contentTypeURL, + "preload": contentTypePlain, + "pubdate": contentTypePlain, + "radiogroup": contentTypePlain, + "readonly": contentTypePlain, + "rel": contentTypeUnsafe, + "required": contentTypePlain, + "reversed": contentTypePlain, + "rows": contentTypePlain, + "rowspan": contentTypePlain, + "sandbox": contentTypeUnsafe, + "spellcheck": contentTypePlain, + "scope": contentTypePlain, + "scoped": contentTypePlain, + "seamless": contentTypePlain, + "selected": contentTypePlain, + "shape": contentTypePlain, + "size": contentTypePlain, + "sizes": contentTypePlain, + "span": contentTypePlain, + "src": contentTypeURL, + "srcdoc": contentTypeHTML, + "srclang": contentTypePlain, + "start": contentTypePlain, + "step": contentTypePlain, + "style": contentTypeCSS, + "tabindex": contentTypePlain, + "target": contentTypePlain, + "title": contentTypePlain, + "type": contentTypeUnsafe, + "usemap": contentTypeURL, + "value": contentTypeUnsafe, + "width": contentTypePlain, + "wrap": contentTypePlain, + "xmlns": contentTypeURL, +} + +// attrType returns a conservative (upper-bound on authority) guess at the +// type of the named attribute. +func attrType(name string) contentType { + name = strings.ToLower(name) + if strings.HasPrefix(name, "data-") { + // Strip data- so that custom attribute heuristics below are + // widely applied. + // Treat data-action as URL below. + name = name[5:] + } else if colon := strings.IndexRune(name, ':'); colon != -1 { + if name[:colon] == "xmlns" { + return contentTypeURL + } + // Treat svg:href and xlink:href as href below. + name = name[colon+1:] + } + if t, ok := attrTypeMap[name]; ok { + return t + } + // Treat partial event handler names as script. + if strings.HasPrefix(name, "on") { + return contentTypeJS + } + + // Heuristics to prevent "javascript:..." injection in custom + // data attributes and custom attributes like g:tweetUrl. + // http://www.w3.org/TR/html5/elements.html#embedding-custom-non-visible-data-with-the-data-attributes: + // "Custom data attributes are intended to store custom data + // private to the page or application, for which there are no + // more appropriate attributes or elements." + // Developers seem to store URL content in data URLs that start + // or end with "URI" or "URL". + if strings.Contains(name, "src") || + strings.Contains(name, "uri") || + strings.Contains(name, "url") { + return contentTypeURL + } + return contentTypePlain +} diff --git a/libgo/go/exp/template/html/clone.go b/libgo/go/exp/template/html/clone.go new file mode 100644 index 0000000..803a64d --- /dev/null +++ b/libgo/go/exp/template/html/clone.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "template/parse" +) + +// clone clones a template Node. +func clone(n parse.Node) parse.Node { + switch t := n.(type) { + case *parse.ActionNode: + return cloneAction(t) + case *parse.IfNode: + b := new(parse.IfNode) + copyBranch(&b.BranchNode, &t.BranchNode) + return b + case *parse.ListNode: + return cloneList(t) + case *parse.RangeNode: + b := new(parse.RangeNode) + copyBranch(&b.BranchNode, &t.BranchNode) + return b + case *parse.TemplateNode: + return cloneTemplate(t) + case *parse.TextNode: + return cloneText(t) + case *parse.WithNode: + b := new(parse.WithNode) + copyBranch(&b.BranchNode, &t.BranchNode) + return b + } + panic("cloning " + n.String() + " is unimplemented") +} + +// cloneAction returns a deep clone of n. +func cloneAction(n *parse.ActionNode) *parse.ActionNode { + // We use keyless fields because they won't compile if a field is added. + return &parse.ActionNode{n.NodeType, n.Line, clonePipe(n.Pipe)} +} + +// cloneList returns a deep clone of n. +func cloneList(n *parse.ListNode) *parse.ListNode { + if n == nil { + return nil + } + // We use keyless fields because they won't compile if a field is added. + c := parse.ListNode{n.NodeType, make([]parse.Node, len(n.Nodes))} + for i, child := range n.Nodes { + c.Nodes[i] = clone(child) + } + return &c +} + +// clonePipe returns a shallow clone of n. +// The escaper does not modify pipe descendants in place so there's no need to +// clone deeply. +func clonePipe(n *parse.PipeNode) *parse.PipeNode { + if n == nil { + return nil + } + // We use keyless fields because they won't compile if a field is added. + return &parse.PipeNode{n.NodeType, n.Line, n.Decl, n.Cmds} +} + +// cloneTemplate returns a deep clone of n. +func cloneTemplate(n *parse.TemplateNode) *parse.TemplateNode { + // We use keyless fields because they won't compile if a field is added. + return &parse.TemplateNode{n.NodeType, n.Line, n.Name, clonePipe(n.Pipe)} +} + +// cloneText clones the given node sharing its []byte. +func cloneText(n *parse.TextNode) *parse.TextNode { + // We use keyless fields because they won't compile if a field is added. + return &parse.TextNode{n.NodeType, n.Text} +} + +// copyBranch clones src into dst. +func copyBranch(dst, src *parse.BranchNode) { + // We use keyless fields because they won't compile if a field is added. + *dst = parse.BranchNode{ + src.NodeType, + src.Line, + clonePipe(src.Pipe), + cloneList(src.List), + cloneList(src.ElseList), + } +} diff --git a/libgo/go/exp/template/html/clone_test.go b/libgo/go/exp/template/html/clone_test.go new file mode 100644 index 0000000..d915425 --- /dev/null +++ b/libgo/go/exp/template/html/clone_test.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "template" + "template/parse" + "testing" +) + +func TestClone(t *testing.T) { + tests := []struct { + input, want, wantClone string + }{ + { + `Hello, {{if true}}{{"<World>"}}{{end}}!`, + "Hello, <World>!", + "Hello, <World>!", + }, + { + `Hello, {{if false}}{{.X}}{{else}}{{"<World>"}}{{end}}!`, + "Hello, <World>!", + "Hello, <World>!", + }, + { + `Hello, {{with "<World>"}}{{.}}{{end}}!`, + "Hello, <World>!", + "Hello, <World>!", + }, + { + `{{range .}}<p>{{.}}</p>{{end}}`, + "<p>foo</p><p><bar></p><p>baz</p>", + "<p>foo</p><p><bar></p><p>baz</p>", + }, + { + `Hello, {{"<World>" | html}}!`, + "Hello, <World>!", + "Hello, <World>!", + }, + { + `Hello{{if 1}}, World{{else}}{{template "d"}}{{end}}!`, + "Hello, World!", + "Hello, World!", + }, + } + + for _, test := range tests { + s := template.Must(template.New("s").Parse(test.input)) + d := template.New("d") + d.Tree = &parse.Tree{Name: d.Name(), Root: cloneList(s.Root)} + + if want, got := s.Root.String(), d.Root.String(); want != got { + t.Errorf("want %q, got %q", want, got) + } + + d, err := Escape(d) + if err != nil { + t.Errorf("%q: failed to escape: %s", test.input, err) + continue + } + + if want, got := "s", s.Name(); want != got { + t.Errorf("want %q, got %q", want, got) + continue + } + if want, got := "d", d.Name(); want != got { + t.Errorf("want %q, got %q", want, got) + continue + } + + data := []string{"foo", "<bar>", "baz"} + + // Make sure escaping d did not affect s. + var b bytes.Buffer + s.Execute(&b, data) + if got := b.String(); got != test.want { + t.Errorf("%q: want %q, got %q", test.input, test.want, got) + continue + } + + b.Reset() + d.Execute(&b, data) + if got := b.String(); got != test.wantClone { + t.Errorf("%q: want %q, got %q", test.input, test.wantClone, got) + } + } +} diff --git a/libgo/go/exp/template/html/content.go b/libgo/go/exp/template/html/content.go new file mode 100644 index 0000000..dcaff8c --- /dev/null +++ b/libgo/go/exp/template/html/content.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// Strings of content from a trusted source. +type ( + // CSS encapsulates known safe content that matches any of: + // (1) The CSS3 stylesheet production, such as `p { color: purple }`. + // (2) The CSS3 rule production, such as `a[href=~"https:"].foo#bar`. + // (3) CSS3 declaration productions, such as `color: red; margin: 2px`. + // (4) The CSS3 value production, such as `rgba(0, 0, 255, 127)`. + // See http://www.w3.org/TR/css3-syntax/#style + CSS string + + // HTML encapsulates a known safe HTML document fragment. + // It should not be used for HTML from a third-party, or HTML with + // unclosed tags or comments. The outputs of a sound HTML sanitizer + // and a template escaped by this package are fine for use with HTML. + HTML string + + // HTMLAttr encapsulates an HTML attribute from a trusted source, + // for example: ` dir="ltr"`. + HTMLAttr string + + // JS encapsulates a known safe EcmaScript5 Expression, or example, + // `(x + y * z())`. + // Template authors are responsible for ensuring that typed expressions + // do not break the intended precedence and that there is no + // statement/expression ambiguity as when passing an expression like + // "{ foo: bar() }\n['foo']()", which is both a valid Expression and a + // valid Program with a very different meaning. + JS string + + // JSStr encapsulates a sequence of characters meant to be embedded + // between quotes in a JavaScript expression. + // The string must match a series of StringCharacters: + // StringCharacter :: SourceCharacter but not `\` or LineTerminator + // | EscapeSequence + // Note that LineContinuations are not allowed. + // JSStr("foo\\nbar") is fine, but JSStr("foo\\\nbar") is not. + JSStr string + + // URL encapsulates a known safe URL as defined in RFC 3896. + // A URL like `javascript:checkThatFormNotEditedBeforeLeavingPage()` + // from a trusted source should go in the page, but by default dynamic + // `javascript:` URLs are filtered out since they are a frequently + // exploited injection vector. + URL string +) + +type contentType uint8 + +const ( + contentTypePlain contentType = iota + contentTypeCSS + contentTypeHTML + contentTypeHTMLAttr + contentTypeJS + contentTypeJSStr + contentTypeURL + // contentTypeUnsafe is used in attr.go for values that affect how + // embedded content and network messages are formed, vetted, + // or interpreted; or which credentials network messages carry. + contentTypeUnsafe +) + +// stringify converts its arguments to a string and the type of the content. +func stringify(args ...interface{}) (string, contentType) { + if len(args) == 1 { + switch s := args[0].(type) { + case string: + return s, contentTypePlain + case CSS: + return string(s), contentTypeCSS + case HTML: + return string(s), contentTypeHTML + case HTMLAttr: + return string(s), contentTypeHTMLAttr + case JS: + return string(s), contentTypeJS + case JSStr: + return string(s), contentTypeJSStr + case URL: + return string(s), contentTypeURL + } + } + return fmt.Sprint(args...), contentTypePlain +} diff --git a/libgo/go/exp/template/html/content_test.go b/libgo/go/exp/template/html/content_test.go new file mode 100644 index 0000000..033dee1 --- /dev/null +++ b/libgo/go/exp/template/html/content_test.go @@ -0,0 +1,222 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "strings" + "template" + "testing" +) + +func TestTypedContent(t *testing.T) { + data := []interface{}{ + `<b> "foo%" O'Reilly &bar;`, + CSS(`a[href =~ "//example.com"]#foo`), + HTML(`Hello, <b>World</b> &tc!`), + HTMLAttr(` dir="ltr"`), + JS(`c && alert("Hello, World!");`), + JSStr(`Hello, World & O'Reilly\x21`), + URL(`greeting=H%69&addressee=(World)`), + } + + // For each content sensitive escaper, see how it does on + // each of the typed strings above. + tests := []struct { + // A template containing a single {{.}}. + input string + want []string + }{ + { + `<style>{{.}} { color: blue }</style>`, + []string{ + `ZgotmplZ`, + // Allowed but not escaped. + `a[href =~ "//example.com"]#foo`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + }, + }, + { + `<div style="{{.}}">`, + []string{ + `ZgotmplZ`, + // Allowed and HTML escaped. + `a[href =~ "//example.com"]#foo`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + }, + }, + { + `{{.}}`, + []string{ + `<b> "foo%" O'Reilly &bar;`, + `a[href =~ "//example.com"]#foo`, + // Not escaped. + `Hello, <b>World</b> &tc!`, + ` dir="ltr"`, + `c && alert("Hello, World!");`, + `Hello, World & O'Reilly\x21`, + `greeting=H%69&addressee=(World)`, + }, + }, + { + `<a{{.}}>`, + []string{ + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + // Allowed and HTML escaped. + ` dir="ltr"`, + `ZgotmplZ`, + `ZgotmplZ`, + `ZgotmplZ`, + }, + }, + { + `<a title={{.}}>`, + []string{ + `<b> "foo%" O'Reilly &bar;`, + `a[href =~ "//example.com"]#foo`, + // Tags stripped, spaces escaped, entity not re-escaped. + `Hello, World &tc!`, + ` dir="ltr"`, + `c && alert("Hello, World!");`, + `Hello, World & O'Reilly\x21`, + `greeting=H%69&addressee=(World)`, + }, + }, + { + `<a title='{{.}}'>`, + []string{ + `<b> "foo%" O'Reilly &bar;`, + `a[href =~ "//example.com"]#foo`, + // Tags stripped, entity not re-escaped. + `Hello, World &tc!`, + ` dir="ltr"`, + `c && alert("Hello, World!");`, + `Hello, World & O'Reilly\x21`, + `greeting=H%69&addressee=(World)`, + }, + }, + { + `<textarea>{{.}}</textarea>`, + []string{ + `<b> "foo%" O'Reilly &bar;`, + `a[href =~ "//example.com"]#foo`, + // Angle brackets escaped to prevent injection of close tags, entity not re-escaped. + `Hello, <b>World</b> &tc!`, + ` dir="ltr"`, + `c && alert("Hello, World!");`, + `Hello, World & O'Reilly\x21`, + `greeting=H%69&addressee=(World)`, + }, + }, + { + `<script>alert({{.}})</script>`, + []string{ + `"\u003cb\u003e \"foo%\" O'Reilly &bar;"`, + `"a[href =~ \"//example.com\"]#foo"`, + `"Hello, \u003cb\u003eWorld\u003c/b\u003e &tc!"`, + `" dir=\"ltr\""`, + // Not escaped. + `c && alert("Hello, World!");`, + // Escape sequence not over-escaped. + `"Hello, World & O'Reilly\x21"`, + `"greeting=H%69&addressee=(World)"`, + }, + }, + { + `<button onclick="alert({{.}})">`, + []string{ + `"\u003cb\u003e \"foo%\" O'Reilly &bar;"`, + `"a[href =~ \"//example.com\"]#foo"`, + `"Hello, \u003cb\u003eWorld\u003c/b\u003e &amp;tc!"`, + `" dir=\"ltr\""`, + // Not JS escaped but HTML escaped. + `c && alert("Hello, World!");`, + // Escape sequence not over-escaped. + `"Hello, World & O'Reilly\x21"`, + `"greeting=H%69&addressee=(World)"`, + }, + }, + { + `<script>alert("{{.}}")</script>`, + []string{ + `\x3cb\x3e \x22foo%\x22 O\x27Reilly \x26bar;`, + `a[href =~ \x22\/\/example.com\x22]#foo`, + `Hello, \x3cb\x3eWorld\x3c\/b\x3e \x26amp;tc!`, + ` dir=\x22ltr\x22`, + `c \x26\x26 alert(\x22Hello, World!\x22);`, + // Escape sequence not over-escaped. + `Hello, World \x26 O\x27Reilly\x21`, + `greeting=H%69\x26addressee=(World)`, + }, + }, + { + `<button onclick='alert("{{.}}")'>`, + []string{ + `\x3cb\x3e \x22foo%\x22 O\x27Reilly \x26bar;`, + `a[href =~ \x22\/\/example.com\x22]#foo`, + `Hello, \x3cb\x3eWorld\x3c\/b\x3e \x26amp;tc!`, + ` dir=\x22ltr\x22`, + `c \x26\x26 alert(\x22Hello, World!\x22);`, + // Escape sequence not over-escaped. + `Hello, World \x26 O\x27Reilly\x21`, + `greeting=H%69\x26addressee=(World)`, + }, + }, + { + `<a href="?q={{.}}">`, + []string{ + `%3cb%3e%20%22foo%25%22%20O%27Reilly%20%26bar%3b`, + `a%5bhref%20%3d~%20%22%2f%2fexample.com%22%5d%23foo`, + `Hello%2c%20%3cb%3eWorld%3c%2fb%3e%20%26amp%3btc%21`, + `%20dir%3d%22ltr%22`, + `c%20%26%26%20alert%28%22Hello%2c%20World%21%22%29%3b`, + `Hello%2c%20World%20%26%20O%27Reilly%5cx21`, + // Quotes and parens are escaped but %69 is not over-escaped. HTML escaping is done. + `greeting=H%69&addressee=%28World%29`, + }, + }, + { + `<style>body { background: url('?img={{.}}') }</style>`, + []string{ + `%3cb%3e%20%22foo%25%22%20O%27Reilly%20%26bar%3b`, + `a%5bhref%20%3d~%20%22%2f%2fexample.com%22%5d%23foo`, + `Hello%2c%20%3cb%3eWorld%3c%2fb%3e%20%26amp%3btc%21`, + `%20dir%3d%22ltr%22`, + `c%20%26%26%20alert%28%22Hello%2c%20World%21%22%29%3b`, + `Hello%2c%20World%20%26%20O%27Reilly%5cx21`, + // Quotes and parens are escaped but %69 is not over-escaped. HTML escaping is not done. + `greeting=H%69&addressee=%28World%29`, + }, + }, + } + + for _, test := range tests { + tmpl := template.Must(Escape(template.Must(template.New("x").Parse(test.input)))) + pre := strings.Index(test.input, "{{.}}") + post := len(test.input) - (pre + 5) + var b bytes.Buffer + for i, x := range data { + b.Reset() + if err := tmpl.Execute(&b, x); err != nil { + t.Errorf("%q with %v: %s", test.input, x, err) + continue + } + if want, got := test.want[i], b.String()[pre:b.Len()-post]; want != got { + t.Errorf("%q with %v:\nwant\n\t%q,\ngot\n\t%q\n", test.input, x, want, got) + continue + } + } + } +} diff --git a/libgo/go/exp/template/html/context.go b/libgo/go/exp/template/html/context.go index 4110068..c44df4d 100644 --- a/libgo/go/exp/template/html/context.go +++ b/libgo/go/exp/template/html/context.go @@ -16,56 +16,176 @@ import ( // http://www.w3.org/TR/html5/the-end.html#parsing-html-fragments // where the context element is null. type context struct { - state state - delim delim + state state + delim delim + urlPart urlPart + jsCtx jsCtx + attr attr + element element + err *Error } func (c context) String() string { - return fmt.Sprintf("context{state: %s, delim: %s", c.state, c.delim) + return fmt.Sprintf("{%v %v %v %v %v %v %v}", c.state, c.delim, c.urlPart, c.jsCtx, c.attr, c.element, c.err) } -// eq is true if the two contexts are identical field-wise. +// eq returns whether two contexts are equal. func (c context) eq(d context) bool { - return c.state == d.state && c.delim == d.delim + return c.state == d.state && + c.delim == d.delim && + c.urlPart == d.urlPart && + c.jsCtx == d.jsCtx && + c.attr == d.attr && + c.element == d.element && + c.err == d.err +} + +// mangle produces an identifier that includes a suffix that distinguishes it +// from template names mangled with different contexts. +func (c context) mangle(templateName string) string { + // The mangled name for the default context is the input templateName. + if c.state == stateText { + return templateName + } + s := templateName + "$htmltemplate_" + c.state.String() + if c.delim != 0 { + s += "_" + c.delim.String() + } + if c.urlPart != 0 { + s += "_" + c.urlPart.String() + } + if c.jsCtx != 0 { + s += "_" + c.jsCtx.String() + } + if c.attr != 0 { + s += "_" + c.attr.String() + } + if c.element != 0 { + s += "_" + c.element.String() + } + return s } // state describes a high-level HTML parser state. // -// It bounds the top of the element stack, and by extension the HTML -// insertion mode, but also contains state that does not correspond to -// anything in the HTML5 parsing algorithm because a single token -// production in the HTML grammar may contain embedded actions in a template. -// For instance, the quoted HTML attribute produced by +// It bounds the top of the element stack, and by extension the HTML insertion +// mode, but also contains state that does not correspond to anything in the +// HTML5 parsing algorithm because a single token production in the HTML +// grammar may contain embedded actions in a template. For instance, the quoted +// HTML attribute produced by // <div title="Hello {{.World}}"> // is a single token in HTML's grammar but in a template spans several nodes. type state uint8 const ( - // statePCDATA is parsed character data. An HTML parser is in + // stateText is parsed character data. An HTML parser is in // this state when its parse position is outside an HTML tag, // directive, comment, and special element body. - statePCDATA state = iota + stateText state = iota // stateTag occurs before an HTML attribute or the end of a tag. stateTag - // stateURI occurs inside an HTML attribute whose content is a URI. - stateURI + // stateAttrName occurs inside an attribute name. + // It occurs between the ^'s in ` ^name^ = value`. + stateAttrName + // stateAfterName occurs after an attr name has ended but before any + // equals sign. It occurs between the ^'s in ` name^ ^= value`. + stateAfterName + // stateBeforeValue occurs after the equals sign but before the value. + // It occurs between the ^'s in ` name =^ ^value`. + stateBeforeValue + // stateHTMLCmt occurs inside an <!-- HTML comment -->. + stateHTMLCmt + // stateRCDATA occurs inside an RCDATA element (<textarea> or <title>) + // as described at http://dev.w3.org/html5/spec/syntax.html#elements-0 + stateRCDATA + // stateAttr occurs inside an HTML attribute whose content is text. + stateAttr + // stateURL occurs inside an HTML attribute whose content is a URL. + stateURL + // stateJS occurs inside an event handler or script element. + stateJS + // stateJSDqStr occurs inside a JavaScript double quoted string. + stateJSDqStr + // stateJSSqStr occurs inside a JavaScript single quoted string. + stateJSSqStr + // stateJSRegexp occurs inside a JavaScript regexp literal. + stateJSRegexp + // stateJSBlockCmt occurs inside a JavaScript /* block comment */. + stateJSBlockCmt + // stateJSLineCmt occurs inside a JavaScript // line comment. + stateJSLineCmt + // stateCSS occurs inside a <style> element or style attribute. + stateCSS + // stateCSSDqStr occurs inside a CSS double quoted string. + stateCSSDqStr + // stateCSSSqStr occurs inside a CSS single quoted string. + stateCSSSqStr + // stateCSSDqURL occurs inside a CSS double quoted url("..."). + stateCSSDqURL + // stateCSSSqURL occurs inside a CSS single quoted url('...'). + stateCSSSqURL + // stateCSSURL occurs inside a CSS unquoted url(...). + stateCSSURL + // stateCSSBlockCmt occurs inside a CSS /* block comment */. + stateCSSBlockCmt + // stateCSSLineCmt occurs inside a CSS // line comment. + stateCSSLineCmt // stateError is an infectious error state outside any valid // HTML/CSS/JS construct. stateError ) var stateNames = [...]string{ - statePCDATA: "statePCDATA", - stateTag: "stateTag", - stateURI: "stateURI", - stateError: "stateError", + stateText: "stateText", + stateTag: "stateTag", + stateAttrName: "stateAttrName", + stateAfterName: "stateAfterName", + stateBeforeValue: "stateBeforeValue", + stateHTMLCmt: "stateHTMLCmt", + stateRCDATA: "stateRCDATA", + stateAttr: "stateAttr", + stateURL: "stateURL", + stateJS: "stateJS", + stateJSDqStr: "stateJSDqStr", + stateJSSqStr: "stateJSSqStr", + stateJSRegexp: "stateJSRegexp", + stateJSBlockCmt: "stateJSBlockCmt", + stateJSLineCmt: "stateJSLineCmt", + stateCSS: "stateCSS", + stateCSSDqStr: "stateCSSDqStr", + stateCSSSqStr: "stateCSSSqStr", + stateCSSDqURL: "stateCSSDqURL", + stateCSSSqURL: "stateCSSSqURL", + stateCSSURL: "stateCSSURL", + stateCSSBlockCmt: "stateCSSBlockCmt", + stateCSSLineCmt: "stateCSSLineCmt", + stateError: "stateError", } func (s state) String() string { - if uint(s) < uint(len(stateNames)) { + if int(s) < len(stateNames) { return stateNames[s] } - return fmt.Sprintf("illegal state %d", uint(s)) + return fmt.Sprintf("illegal state %d", int(s)) +} + +// isComment is true for any state that contains content meant for template +// authors & maintainers, not for end-users or machines. +func isComment(s state) bool { + switch s { + case stateHTMLCmt, stateJSBlockCmt, stateJSLineCmt, stateCSSBlockCmt, stateCSSLineCmt: + return true + } + return false +} + +// isInTag return whether s occurs solely inside an HTML tag. +func isInTag(s state) bool { + switch s { + case stateTag, stateAttrName, stateAfterName, stateBeforeValue, stateAttr: + return true + } + return false } // delim is the delimiter that will end the current HTML attribute. @@ -91,8 +211,129 @@ var delimNames = [...]string{ } func (d delim) String() string { - if uint(d) < uint(len(delimNames)) { + if int(d) < len(delimNames) { return delimNames[d] } - return fmt.Sprintf("illegal delim %d", uint(d)) + return fmt.Sprintf("illegal delim %d", int(d)) +} + +// urlPart identifies a part in an RFC 3986 hierarchical URL to allow different +// encoding strategies. +type urlPart uint8 + +const ( + // urlPartNone occurs when not in a URL, or possibly at the start: + // ^ in "^http://auth/path?k=v#frag". + urlPartNone urlPart = iota + // urlPartPreQuery occurs in the scheme, authority, or path; between the + // ^s in "h^ttp://auth/path^?k=v#frag". + urlPartPreQuery + // urlPartQueryOrFrag occurs in the query portion between the ^s in + // "http://auth/path?^k=v#frag^". + urlPartQueryOrFrag + // urlPartUnknown occurs due to joining of contexts both before and + // after the query separator. + urlPartUnknown +) + +var urlPartNames = [...]string{ + urlPartNone: "urlPartNone", + urlPartPreQuery: "urlPartPreQuery", + urlPartQueryOrFrag: "urlPartQueryOrFrag", + urlPartUnknown: "urlPartUnknown", +} + +func (u urlPart) String() string { + if int(u) < len(urlPartNames) { + return urlPartNames[u] + } + return fmt.Sprintf("illegal urlPart %d", int(u)) +} + +// jsCtx determines whether a '/' starts a regular expression literal or a +// division operator. +type jsCtx uint8 + +const ( + // jsCtxRegexp occurs where a '/' would start a regexp literal. + jsCtxRegexp jsCtx = iota + // jsCtxDivOp occurs where a '/' would start a division operator. + jsCtxDivOp + // jsCtxUnknown occurs where a '/' is ambiguous due to context joining. + jsCtxUnknown +) + +func (c jsCtx) String() string { + switch c { + case jsCtxRegexp: + return "jsCtxRegexp" + case jsCtxDivOp: + return "jsCtxDivOp" + case jsCtxUnknown: + return "jsCtxUnknown" + } + return fmt.Sprintf("illegal jsCtx %d", int(c)) +} + +// element identifies the HTML element when inside a start tag or special body. +// Certain HTML element (for example <script> and <style>) have bodies that are +// treated differently from stateText so the element type is necessary to +// transition into the correct context at the end of a tag and to identify the +// end delimiter for the body. +type element uint8 + +const ( + // elementNone occurs outside a special tag or special element body. + elementNone element = iota + // elementScript corresponds to the raw text <script> element. + elementScript + // elementStyle corresponds to the raw text <style> element. + elementStyle + // elementTextarea corresponds to the RCDATA <textarea> element. + elementTextarea + // elementTitle corresponds to the RCDATA <title> element. + elementTitle +) + +var elementNames = [...]string{ + elementNone: "elementNone", + elementScript: "elementScript", + elementStyle: "elementStyle", + elementTextarea: "elementTextarea", + elementTitle: "elementTitle", +} + +func (e element) String() string { + if int(e) < len(elementNames) { + return elementNames[e] + } + return fmt.Sprintf("illegal element %d", int(e)) +} + +// attr identifies the most recent HTML attribute when inside a start tag. +type attr uint8 + +const ( + // attrNone corresponds to a normal attribute or no attribute. + attrNone attr = iota + // attrScript corresponds to an event handler attribute. + attrScript + // attrStyle corresponds to the style attribute whose value is CSS. + attrStyle + // attrURL corresponds to an attribute whose value is a URL. + attrURL +) + +var attrNames = [...]string{ + attrNone: "attrNone", + attrScript: "attrScript", + attrStyle: "attrStyle", + attrURL: "attrURL", +} + +func (a attr) String() string { + if int(a) < len(attrNames) { + return attrNames[a] + } + return fmt.Sprintf("illegal attr %d", int(a)) } diff --git a/libgo/go/exp/template/html/css.go b/libgo/go/exp/template/html/css.go new file mode 100644 index 0000000..c22ec6d --- /dev/null +++ b/libgo/go/exp/template/html/css.go @@ -0,0 +1,268 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "fmt" + "unicode" + "utf8" +) + +// endsWithCSSKeyword returns whether b ends with an ident that +// case-insensitively matches the lower-case kw. +func endsWithCSSKeyword(b []byte, kw string) bool { + i := len(b) - len(kw) + if i < 0 { + // Too short. + return false + } + if i != 0 { + r, _ := utf8.DecodeLastRune(b[:i]) + if isCSSNmchar(r) { + // Too long. + return false + } + } + // Many CSS keywords, such as "!important" can have characters encoded, + // but the URI production does not allow that according to + // http://www.w3.org/TR/css3-syntax/#TOK-URI + // This does not attempt to recognize encoded keywords. For example, + // given "\75\72\6c" and "url" this return false. + return string(bytes.ToLower(b[i:])) == kw +} + +// isCSSNmchar returns whether rune is allowed anywhere in a CSS identifier. +func isCSSNmchar(rune int) bool { + // Based on the CSS3 nmchar production but ignores multi-rune escape + // sequences. + // http://www.w3.org/TR/css3-syntax/#SUBTOK-nmchar + return 'a' <= rune && rune <= 'z' || + 'A' <= rune && rune <= 'Z' || + '0' <= rune && rune <= '9' || + '-' == rune || + '_' == rune || + // Non-ASCII cases below. + 0x80 <= rune && rune <= 0xd7ff || + 0xe000 <= rune && rune <= 0xfffd || + 0x10000 <= rune && rune <= 0x10ffff +} + +// decodeCSS decodes CSS3 escapes given a sequence of stringchars. +// If there is no change, it returns the input, otherwise it returns a slice +// backed by a new array. +// http://www.w3.org/TR/css3-syntax/#SUBTOK-stringchar defines stringchar. +func decodeCSS(s []byte) []byte { + i := bytes.IndexByte(s, '\\') + if i == -1 { + return s + } + // The UTF-8 sequence for a codepoint is never longer than 1 + the + // number hex digits need to represent that codepoint, so len(s) is an + // upper bound on the output length. + b := make([]byte, 0, len(s)) + for len(s) != 0 { + i := bytes.IndexByte(s, '\\') + if i == -1 { + i = len(s) + } + b, s = append(b, s[:i]...), s[i:] + if len(s) < 2 { + break + } + // http://www.w3.org/TR/css3-syntax/#SUBTOK-escape + // escape ::= unicode | '\' [#x20-#x7E#x80-#xD7FF#xE000-#xFFFD#x10000-#x10FFFF] + if isHex(s[1]) { + // http://www.w3.org/TR/css3-syntax/#SUBTOK-unicode + // unicode ::= '\' [0-9a-fA-F]{1,6} wc? + j := 2 + for j < len(s) && j < 7 && isHex(s[j]) { + j++ + } + rune := hexDecode(s[1:j]) + if rune > unicode.MaxRune { + rune, j = rune/16, j-1 + } + n := utf8.EncodeRune(b[len(b):cap(b)], rune) + // The optional space at the end allows a hex + // sequence to be followed by a literal hex. + // string(decodeCSS([]byte(`\A B`))) == "\nB" + b, s = b[:len(b)+n], skipCSSSpace(s[j:]) + } else { + // `\\` decodes to `\` and `\"` to `"`. + _, n := utf8.DecodeRune(s[1:]) + b, s = append(b, s[1:1+n]...), s[1+n:] + } + } + return b +} + +// isHex returns whether the given character is a hex digit. +func isHex(c byte) bool { + return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' +} + +// hexDecode decodes a short hex digit sequence: "10" -> 16. +func hexDecode(s []byte) int { + n := 0 + for _, c := range s { + n <<= 4 + switch { + case '0' <= c && c <= '9': + n |= int(c - '0') + case 'a' <= c && c <= 'f': + n |= int(c-'a') + 10 + case 'A' <= c && c <= 'F': + n |= int(c-'A') + 10 + default: + panic(fmt.Sprintf("Bad hex digit in %q", s)) + } + } + return n +} + +// skipCSSSpace returns a suffix of c, skipping over a single space. +func skipCSSSpace(c []byte) []byte { + if len(c) == 0 { + return c + } + // wc ::= #x9 | #xA | #xC | #xD | #x20 + switch c[0] { + case '\t', '\n', '\f', ' ': + return c[1:] + case '\r': + // This differs from CSS3's wc production because it contains a + // probable spec error whereby wc contains all the single byte + // sequences in nl (newline) but not CRLF. + if len(c) >= 2 && c[1] == '\n' { + return c[2:] + } + return c[1:] + } + return c +} + +// isCSSSpace returns whether b is a CSS space char as defined in wc. +func isCSSSpace(b byte) bool { + switch b { + case '\t', '\n', '\f', '\r', ' ': + return true + } + return false +} + +// cssEscaper escapes HTML and CSS special characters using \<hex>+ escapes. +func cssEscaper(args ...interface{}) string { + s, _ := stringify(args...) + var b bytes.Buffer + written := 0 + for i, r := range s { + var repl string + switch r { + case 0: + repl = `\0` + case '\t': + repl = `\9` + case '\n': + repl = `\a` + case '\f': + repl = `\c` + case '\r': + repl = `\d` + // Encode HTML specials as hex so the output can be embedded + // in HTML attributes without further encoding. + case '"': + repl = `\22` + case '&': + repl = `\26` + case '\'': + repl = `\27` + case '(': + repl = `\28` + case ')': + repl = `\29` + case '+': + repl = `\2b` + case '/': + repl = `\2f` + case ':': + repl = `\3a` + case ';': + repl = `\3b` + case '<': + repl = `\3c` + case '>': + repl = `\3e` + case '\\': + repl = `\\` + case '{': + repl = `\7b` + case '}': + repl = `\7d` + default: + continue + } + b.WriteString(s[written:i]) + b.WriteString(repl) + written = i + utf8.RuneLen(r) + if repl != `\\` && (written == len(s) || isHex(s[written]) || isCSSSpace(s[written])) { + b.WriteByte(' ') + } + } + if written == 0 { + return s + } + b.WriteString(s[written:]) + return b.String() +} + +var expressionBytes = []byte("expression") +var mozBindingBytes = []byte("mozbinding") + +// cssValueFilter allows innocuous CSS values in the output including CSS +// quantities (10px or 25%), ID or class literals (#foo, .bar), keyword values +// (inherit, blue), and colors (#888). +// It filters out unsafe values, such as those that affect token boundaries, +// and anything that might execute scripts. +func cssValueFilter(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeCSS { + return s + } + b, id := decodeCSS([]byte(s)), make([]byte, 0, 64) + + // CSS3 error handling is specified as honoring string boundaries per + // http://www.w3.org/TR/css3-syntax/#error-handling : + // Malformed declarations. User agents must handle unexpected + // tokens encountered while parsing a declaration by reading until + // the end of the declaration, while observing the rules for + // matching pairs of (), [], {}, "", and '', and correctly handling + // escapes. For example, a malformed declaration may be missing a + // property, colon (:) or value. + // So we need to make sure that values do not have mismatched bracket + // or quote characters to prevent the browser from restarting parsing + // inside a string that might embed JavaScript source. + for i, c := range b { + switch c { + case 0, '"', '\'', '(', ')', '/', ';', '@', '[', '\\', ']', '`', '{', '}': + return filterFailsafe + case '-': + // Disallow <!-- or -->. + // -- should not appear in valid identifiers. + if i != 0 && '-' == b[i-1] { + return filterFailsafe + } + default: + if c < 0x80 && isCSSNmchar(int(c)) { + id = append(id, c) + } + } + } + id = bytes.ToLower(id) + if bytes.Index(id, expressionBytes) != -1 || bytes.Index(id, mozBindingBytes) != -1 { + return filterFailsafe + } + return string(b) +} diff --git a/libgo/go/exp/template/html/css_test.go b/libgo/go/exp/template/html/css_test.go new file mode 100644 index 0000000..5f633e8 --- /dev/null +++ b/libgo/go/exp/template/html/css_test.go @@ -0,0 +1,281 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strconv" + "strings" + "testing" +) + +func TestEndsWithCSSKeyword(t *testing.T) { + tests := []struct { + css, kw string + want bool + }{ + {"", "url", false}, + {"url", "url", true}, + {"URL", "url", true}, + {"Url", "url", true}, + {"url", "important", false}, + {"important", "important", true}, + {"image-url", "url", false}, + {"imageurl", "url", false}, + {"image url", "url", true}, + } + for _, test := range tests { + got := endsWithCSSKeyword([]byte(test.css), test.kw) + if got != test.want { + t.Errorf("want %t but got %t for css=%v, kw=%v", test.want, got, test.css, test.kw) + } + } +} + +func TestIsCSSNmchar(t *testing.T) { + tests := []struct { + rune int + want bool + }{ + {0, false}, + {'0', true}, + {'9', true}, + {'A', true}, + {'Z', true}, + {'a', true}, + {'z', true}, + {'_', true}, + {'-', true}, + {':', false}, + {';', false}, + {' ', false}, + {0x7f, false}, + {0x80, true}, + {0x1234, true}, + {0xd800, false}, + {0xdc00, false}, + {0xfffe, false}, + {0x10000, true}, + {0x110000, false}, + } + for _, test := range tests { + got := isCSSNmchar(test.rune) + if got != test.want { + t.Errorf("%q: want %t but got %t", string(test.rune), test.want, got) + } + } +} + +func TestDecodeCSS(t *testing.T) { + tests := []struct { + css, want string + }{ + {``, ``}, + {`foo`, `foo`}, + {`foo\`, `foo`}, + {`foo\\`, `foo\`}, + {`\`, ``}, + {`\A`, "\n"}, + {`\a`, "\n"}, + {`\0a`, "\n"}, + {`\00000a`, "\n"}, + {`\000000a`, "\u0000a"}, + {`\1234 5`, "\u1234" + "5"}, + {`\1234\20 5`, "\u1234" + " 5"}, + {`\1234\A 5`, "\u1234" + "\n5"}, + {"\\1234\t5", "\u1234" + "5"}, + {"\\1234\n5", "\u1234" + "5"}, + {"\\1234\r\n5", "\u1234" + "5"}, + {`\12345`, "\U00012345"}, + {`\\`, `\`}, + {`\\ `, `\ `}, + {`\"`, `"`}, + {`\'`, `'`}, + {`\.`, `.`}, + {`\. .`, `. .`}, + { + `The \3c i\3equick\3c/i\3e,\d\A\3cspan style=\27 color:brown\27\3e brown\3c/span\3e fox jumps\2028over the \3c canine class=\22lazy\22 \3e dog\3c/canine\3e`, + "The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>", + }, + } + for _, test := range tests { + got1 := string(decodeCSS([]byte(test.css))) + if got1 != test.want { + t.Errorf("%q: want\n\t%q\nbut got\n\t%q", test.css, test.want, got1) + } + recoded := cssEscaper(got1) + if got2 := string(decodeCSS([]byte(recoded))); got2 != test.want { + t.Errorf("%q: escape & decode not dual for %q", test.css, recoded) + } + } +} + +func TestHexDecode(t *testing.T) { + for i := 0; i < 0x200000; i += 101 /* coprime with 16 */ { + s := strconv.Itob(i, 16) + if got := hexDecode([]byte(s)); got != i { + t.Errorf("%s: want %d but got %d", s, i, got) + } + s = strings.ToUpper(s) + if got := hexDecode([]byte(s)); got != i { + t.Errorf("%s: want %d but got %d", s, i, got) + } + } +} + +func TestSkipCSSSpace(t *testing.T) { + tests := []struct { + css, want string + }{ + {"", ""}, + {"foo", "foo"}, + {"\n", ""}, + {"\r\n", ""}, + {"\r", ""}, + {"\t", ""}, + {" ", ""}, + {"\f", ""}, + {" foo", "foo"}, + {" foo", " foo"}, + {`\20`, `\20`}, + } + for _, test := range tests { + got := string(skipCSSSpace([]byte(test.css))) + if got != test.want { + t.Errorf("%q: want %q but got %q", test.css, test.want, got) + } + } +} + +func TestCSSEscaper(t *testing.T) { + input := ("\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./` + + `0123456789:;<=>?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\]^_` + + "`abcdefghijklmno" + + "pqrstuvwxyz{|}~\x7f" + + "\u00A0\u0100\u2028\u2029\ufeff\U0001D11E") + + want := ("\\0\x01\x02\x03\x04\x05\x06\x07" + + "\x08\\9 \\a\x0b\\c \\d\x0E\x0F" + + "\x10\x11\x12\x13\x14\x15\x16\x17" + + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !\22#$%\26\27\28\29*\2b,-.\2f ` + + `0123456789\3a\3b\3c=\3e?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\\]^_` + + "`abcdefghijklmno" + + `pqrstuvwxyz\7b|\7d~` + "\u007f" + + "\u00A0\u0100\u2028\u2029\ufeff\U0001D11E") + + got := cssEscaper(input) + if got != want { + t.Errorf("encode: want\n\t%q\nbut got\n\t%q", want, got) + } + + got = string(decodeCSS([]byte(got))) + if input != got { + t.Errorf("decode: want\n\t%q\nbut got\n\t%q", input, got) + } +} + +func TestCSSValueFilter(t *testing.T) { + tests := []struct { + css, want string + }{ + {"", ""}, + {"foo", "foo"}, + {"0", "0"}, + {"0px", "0px"}, + {"-5px", "-5px"}, + {"1.25in", "1.25in"}, + {"+.33em", "+.33em"}, + {"100%", "100%"}, + {"12.5%", "12.5%"}, + {".foo", ".foo"}, + {"#bar", "#bar"}, + {"corner-radius", "corner-radius"}, + {"-moz-corner-radius", "-moz-corner-radius"}, + {"#000", "#000"}, + {"#48f", "#48f"}, + {"#123456", "#123456"}, + {"U+00-FF, U+980-9FF", "U+00-FF, U+980-9FF"}, + {"color: red", "color: red"}, + {"<!--", "ZgotmplZ"}, + {"-->", "ZgotmplZ"}, + {"<![CDATA[", "ZgotmplZ"}, + {"]]>", "ZgotmplZ"}, + {"</style", "ZgotmplZ"}, + {`"`, "ZgotmplZ"}, + {`'`, "ZgotmplZ"}, + {"`", "ZgotmplZ"}, + {"\x00", "ZgotmplZ"}, + {"/* foo */", "ZgotmplZ"}, + {"//", "ZgotmplZ"}, + {"[href=~", "ZgotmplZ"}, + {"expression(alert(1337))", "ZgotmplZ"}, + {"-expression(alert(1337))", "ZgotmplZ"}, + {"expression", "ZgotmplZ"}, + {"Expression", "ZgotmplZ"}, + {"EXPRESSION", "ZgotmplZ"}, + {"-moz-binding", "ZgotmplZ"}, + {"-expr\x00ession(alert(1337))", "ZgotmplZ"}, + {`-expr\0ession(alert(1337))`, "ZgotmplZ"}, + {`-express\69on(alert(1337))`, "ZgotmplZ"}, + {`-express\69 on(alert(1337))`, "ZgotmplZ"}, + {`-exp\72 ession(alert(1337))`, "ZgotmplZ"}, + {`-exp\52 ession(alert(1337))`, "ZgotmplZ"}, + {`-exp\000052 ession(alert(1337))`, "ZgotmplZ"}, + {`-expre\0000073sion`, "-expre\x073sion"}, + {`@import url evil.css`, "ZgotmplZ"}, + } + for _, test := range tests { + got := cssValueFilter(test.css) + if got != test.want { + t.Errorf("%q: want %q but got %q", test.css, test.want, got) + } + } +} + +func BenchmarkCSSEscaper(b *testing.B) { + for i := 0; i < b.N; i++ { + cssEscaper("The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>") + } +} + +func BenchmarkCSSEscaperNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + cssEscaper("The quick, brown fox jumps over the lazy dog.") + } +} + +func BenchmarkDecodeCSS(b *testing.B) { + s := []byte(`The \3c i\3equick\3c/i\3e,\d\A\3cspan style=\27 color:brown\27\3e brown\3c/span\3e fox jumps\2028over the \3c canine class=\22lazy\22 \3edog\3c/canine\3e`) + b.ResetTimer() + for i := 0; i < b.N; i++ { + decodeCSS(s) + } +} + +func BenchmarkDecodeCSSNoSpecials(b *testing.B) { + s := []byte("The quick, brown fox jumps over the lazy dog.") + b.ResetTimer() + for i := 0; i < b.N; i++ { + decodeCSS(s) + } +} + +func BenchmarkCSSValueFilter(b *testing.B) { + for i := 0; i < b.N; i++ { + cssValueFilter(` e\78preS\0Sio/**/n(alert(1337))`) + } +} + +func BenchmarkCSSValueFilterOk(b *testing.B) { + for i := 0; i < b.N; i++ { + cssValueFilter(`Times New Roman`) + } +} diff --git a/libgo/go/exp/template/html/doc.go b/libgo/go/exp/template/html/doc.go new file mode 100644 index 0000000..a9b78ca --- /dev/null +++ b/libgo/go/exp/template/html/doc.go @@ -0,0 +1,190 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package html is a specialization of package template that automates the +construction of HTML output that is safe against code injection. + + +Introduction + +To use this package, invoke the standard template package to parse a template +set, and then use this package’s EscapeSet function to secure the set. +The arguments to EscapeSet are the template set and the names of all templates +that will be passed to Execute. + + set, err := new(template.Set).Parse(...) + set, err = EscapeSet(set, "templateName0", ...) + +If successful, set will now be injection-safe. Otherwise, the returned set will +be nil and an error, described below, will explain the problem. + +The template names do not need to include helper templates but should include +all names x used thus: + + set.Execute(out, x, ...) + +EscapeSet modifies the named templates in place to treat data values as plain +text safe for embedding in an HTML document. The escaping is contextual, so +actions can appear within JavaScript, CSS, and URI contexts without introducing'hazards. + +The security model used by this package assumes that template authors are +trusted, while Execute's data parameter is not. More details are provided below. + +Example + + tmpls, err := new(template.Set).Parse(`{{define "t'}}Hello, {{.}}!{{end}}`) + +when used by itself + + tmpls.Execute(out, "t", "<script>alert('you have been pwned')</script>") + +produces + + Hello, <script>alert('you have been pwned')</script>! + +but after securing with EscapeSet like this, + + tmpls, err := EscapeSet(tmpls, "t") + tmpls.Execute(out, "t", ...) + +produces the safe, escaped HTML output + + Hello, <script>alert('you have been pwned')</script>! + + +Contexts + +EscapeSet understands HTML, CSS, JavaScript, and URIs. It adds sanitizing +functions to each simple action pipeline, so given the excerpt + + <a href="/search?q={{.}}">{{.}}</a> + +EscapeSet will rewrite each {{.}} to add escaping functions where necessary, +in this case, + + <a href="/search?q={{. | urlquery}}">{{. | html}}</a> + + +Errors + +See the documentation of ErrorCode for details. + + +A fuller picture + +The rest of this package comment may be skipped on first reading; it includes +details necessary to understand escaping contexts and error messages. Most users +will not need to understand these details. + + +Contexts + +Assuming {{.}} is `O'Reilly: How are <i>you</i>?`, the table below shows +how {{.}} appears when used in the context to the left. + +Context {{.}} After +{{.}} O'Reilly: How are <i>you</i>? +<a title='{{.}}'> O'Reilly: How are you? +<a href="/{{.}}"> O'Reilly: How are %3ci%3eyou%3c/i%3e? +<a href="?q={{.}}"> O'Reilly%3a%20How%20are%3ci%3e...%3f +<a onx='f("{{.}}")'> O\x27Reilly: How are \x3ci\x3eyou...? +<a onx='f({{.}})'> "O\x27Reilly: How are \x3ci\x3eyou...?" +<a onx='pattern = /{{.}}/;'> O\x27Reilly: How are \x3ci\x3eyou...\x3f + +If used in an unsafe context, then the value might be filtered out: + +Context {{.}} After +<a href="{{.}}"> #ZgotmplZ + +since "O'Reilly:" is not an allowed protocol like "http:". + + +If {{.}} is the innocuous word, `left`, then it can appear more widely, + +Context {{.}} After +{{.}} left +<a title='{{.}}'> left +<a href='{{.}}'> left +<a href='/{{.}}'> left +<a href='?dir={{.}}'> left +<a style="border-{{.}}: 4px"> left +<a style="align: {{.}}"> left +<a style="background: '{{.}}'> left +<a style="background: url('{{.}}')> left +<style>p.{{.}} {color:red}</style> left + +Non-string values can be used in JavaScript contexts. +If {{.}} is + + []struct{A,B string}{ "foo", "bar" } + +in the escaped template + + <script>var pair = {{.}};</script> + +then the template output is + + <script>var pair = {"A": "foo", "B": "bar"};</script> + +See package json to understand how non-string content is marshalled for +embedding in JavaScript contexts. + + +Typed Strings + +By default, EscapeSet assumes all pipelines produce a plain text string. It +adds escaping pipeline stages necessary to correctly and safely embed that +plain text string in the appropriate context. + +When a data value is not plain text, you can make sure it is not over-escaped +by marking it with its type. + +Types HTML, JS, URL, and others from content.go can carry safe content that is +exempted from escaping. + +The template + + Hello, {{.}}! + +can be invoked with + + tmpl.Execute(out, HTML(`<b>World</b>`)) + +to produce + + Hello, <b>World</b>! + +instead of the + + Hello, <b>World<b>! + +that would have been produced if {{.}} was a regular string. + + +Security Model + +http://js-quasis-libraries-and-repl.googlecode.com/svn/trunk/safetemplate.html#problem_definition defines "safe" as used by this package. + +This package assumes that template authors are trusted, that Execute's data +parameter is not, and seeks to preserve the properties below in the face +of untrusted data: + +Structure Preservation Property +"... when a template author writes an HTML tag in a safe templating language, +the browser will interpret the corresponding portion of the output as a tag +regardless of the values of untrusted data, and similarly for other structures +such as attribute boundaries and JS and CSS string boundaries." + +Code Effect Property +"... only code specified by the template author should run as a result of +injecting the template output into a page and all code specified by the +template author should run as a result of the same." + +Least Surprise Property +"A developer (or code reviewer) familiar with HTML, CSS, and JavaScript; +who knows that EscapeSet is applied should be able to look at a {{.}} +and correctly infer what sanitization happens." +*/ +package html diff --git a/libgo/go/exp/template/html/error.go b/libgo/go/exp/template/html/error.go new file mode 100644 index 0000000..5515bfe --- /dev/null +++ b/libgo/go/exp/template/html/error.go @@ -0,0 +1,213 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// Error describes a problem encountered during template Escaping. +type Error struct { + // ErrorCode describes the kind of error. + ErrorCode ErrorCode + // Name is the name of the template in which the error was encountered. + Name string + // Line is the line number of the error in the template source or 0. + Line int + // Description is a human-readable description of the problem. + Description string +} + +// ErrorCode is a code for a kind of error. +type ErrorCode int + +// We define codes for each error that manifests while escaping templates, but +// escaped templates may also fail at runtime. +// +// Output: "ZgotmplZ" +// Example: +// <img src="{{.X}}"> +// where {{.X}} evaluates to `javascript:...` +// Discussion: +// "ZgotmplZ" is a special value that indicates that unsafe content reached a +// CSS or URL context at runtime. The output of the example will be +// <img src="#ZgotmplZ"> +// If the data comes from a trusted source, use content types to exempt it +// from filtering: URL(`javascript:...`). +const ( + // OK indicates the lack of an error. + OK ErrorCode = iota + + // ErrAmbigContext: "... appears in an ambiguous URL context" + // Example: + // <a href=" + // {{if .C}} + // /path/ + // {{else}} + // /search?q= + // {{end}} + // {{.X}} + // "> + // Discussion: + // {{.X}} is in an ambiguous URL context since, depending on {{.C}}, + // it may be either a URL suffix or a query parameter. + // Moving {{.X}} into the condition removes the ambiguity: + // <a href="{{if .C}}/path/{{.X}}{{else}}/search?q={{.X}}"> + ErrAmbigContext + + // ErrBadHTML: "expected space, attr name, or end of tag, but got ...", + // "... in unquoted attr", "... in attribute name" + // Example: + // <a href = /search?q=foo> + // <href=foo> + // <form na<e=...> + // <option selected< + // Discussion: + // This is often due to a typo in an HTML element, but some runes + // are banned in tag names, attribute names, and unquoted attribute + // values because they can tickle parser ambiguities. + // Quoting all attributes is the best policy. + ErrBadHTML + + // ErrBranchEnd: "{{if}} branches end in different contexts" + // Example: + // {{if .C}}<a href="{{end}}{{.X}} + // Discussion: + // EscapeSet statically examines each possible path when it encounters + // a {{if}}, {{range}}, or {{with}} to escape any following pipelines. + // The example is ambiguous since {{.X}} might be an HTML text node, + // or a URL prefix in an HTML attribute. EscapeSet needs to understand + // the context of {{.X}} to escape it, but that depends on the + // run-time value of {{.C}}. + // + // The problem is usually something like missing quotes or angle + // brackets, or can be avoided by refactoring to put the two contexts + // into different branches of an if, range or with. If the problem + // is in a {{range}} over a collection that should never be empty, + // adding a dummy {{else}} can help. + ErrBranchEnd + + // ErrEndContext: "... ends in a non-text context: ..." + // Examples: + // <div + // <div title="no close quote> + // <script>f() + // Discussion: + // EscapeSet assumes the ouput is a DocumentFragment of HTML. + // Templates that end without closing tags will trigger this error. + // Templates that produce incomplete Fragments should not be named + // in the call to EscapeSet. + // + // If you have a helper template in your set that is not meant to + // produce a document fragment, then do not pass its name to + // EscapeSet(set, ...names). + // + // {{define "main"}} <script>{{template "helper"}}</script> {{end}} + // {{define "helper"}} document.write(' <div title=" ') {{end}} + // + // "helper" does not produce a valid document fragment, though it does + // produce a valid JavaScript Program. + ErrEndContext + + // ErrNoNames: "must specify names of top level templates" + // + // EscapeSet does not assume that all templates in a set produce HTML. + // Some may be helpers that produce snippets of other languages. + // Passing in no template names is most likely an error, + // so EscapeSet(set) will panic. + // If you call EscapeSet with a slice of names, guard it with len: + // + // if len(names) != 0 { + // set, err := EscapeSet(set, ...names) + // } + ErrNoNames + + // ErrNoSuchTemplate: "no such template ..." + // Examples: + // {{define "main"}}<div {{template "attrs"}}>{{end}} + // {{define "attrs"}}href="{{.URL}}"{{end}} + // Discussion: + // EscapeSet looks through template calls to compute the context. + // Here the {{.URL}} in "attrs" must be treated as a URL when called + // from "main", but if "attrs" is not in set when + // EscapeSet(&set, "main") is called, this error will arise. + ErrNoSuchTemplate + + // ErrOutputContext: "cannot compute output context for template ..." + // Examples: + // {{define "t"}}{{if .T}}{{template "t" .T}}{{end}}{{.H}}",{{end}} + // Discussion: + // A recursive template does not end in the same context in which it + // starts, and a reliable output context cannot be computed. + // Look for typos in the named template. + // If the template should not be called in the named start context, + // look for calls to that template in unexpected contexts. + // Maybe refactor recursive templates to not be recursive. + ErrOutputContext + + // ErrPartialCharset: "unfinished JS regexp charset in ..." + // Example: + // <script>var pattern = /foo[{{.Chars}}]/</script> + // Discussion: + // EscapeSet does not support interpolation into regular expression + // literal character sets. + ErrPartialCharset + + // ErrPartialEscape: "unfinished escape sequence in ..." + // Example: + // <script>alert("\{{.X}}")</script> + // Discussion: + // EscapeSet does not support actions following a backslash. + // This is usually an error and there are better solutions; for + // our example + // <script>alert("{{.X}}")</script> + // should work, and if {{.X}} is a partial escape sequence such as + // "xA0", mark the whole sequence as safe content: JSStr(`\xA0`) + ErrPartialEscape + + // ErrRangeLoopReentry: "on range loop re-entry: ..." + // Example: + // {{range .}}<p class={{.}}{{end}} + // Discussion: + // If an iteration through a range would cause it to end in a + // different context than an earlier pass, there is no single context. + // In the example, the <p> tag is missing a '>'. + // EscapeSet cannot tell whether {{.}} is meant to be an HTML class or + // the content of a broken <p> element and complains because the + // second iteration would produce something like + // + // <p class=foo<p class=bar + ErrRangeLoopReentry + + // ErrSlashAmbig: '/' could start a division or regexp. + // Example: + // <script> + // {{if .C}}var x = 1{{end}} + // /-{{.N}}/i.test(x) ? doThis : doThat(); + // </script> + // Discussion: + // The example above could produce `var x = 1/-2/i.test(s)...` + // in which the first '/' is a mathematical division operator or it + // could produce `/-2/i.test(s)` in which the first '/' starts a + // regexp literal. + // Look for missing semicolons inside branches, and maybe add + // parentheses to make it clear which interpretation you intend. + ErrSlashAmbig +) + +func (e *Error) String() string { + if e.Line != 0 { + return fmt.Sprintf("exp/template/html:%s:%d: %s", e.Name, e.Line, e.Description) + } else if e.Name != "" { + return fmt.Sprintf("exp/template/html:%s: %s", e.Name, e.Description) + } + return "exp/template/html: " + e.Description +} + +// errorf creates an error given a format string f and args. +// The template Name still needs to be supplied. +func errorf(k ErrorCode, line int, f string, args ...interface{}) *Error { + return &Error{k, "", line, fmt.Sprintf(f, args...)} +} diff --git a/libgo/go/exp/template/html/escape.go b/libgo/go/exp/template/html/escape.go index e0e87b9..74abcce 100644 --- a/libgo/go/exp/template/html/escape.go +++ b/libgo/go/exp/template/html/escape.go @@ -2,104 +2,760 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package html is a specialization of exp/template that automates the -// construction of safe HTML output. -// At the moment, the escaping is naive. All dynamic content is assumed to be -// plain text interpolated in an HTML PCDATA context. package html import ( + "bytes" + "fmt" + "html" + "os" "template" "template/parse" ) -// Escape rewrites each action in the template to guarantee the output is -// HTML-escaped. -func Escape(t *template.Template) { - // If the parser shares trees based on common-subexpression - // joining then we will need to avoid multiply escaping the same action. - escapeListNode(t.Tree.Root) +// Escape rewrites each action in the template to guarantee that the output is +// properly escaped. +func Escape(t *template.Template) (*template.Template, os.Error) { + var s template.Set + s.Add(t) + if _, err := EscapeSet(&s, t.Name()); err != nil { + return nil, err + } + // TODO: if s contains cloned dependencies due to self-recursion + // cross-context, error out. + return t, nil } -// escapeNode dispatches to escape<NodeType> helpers by type. -func escapeNode(node parse.Node) { - switch n := node.(type) { - case *parse.ListNode: - escapeListNode(n) - case *parse.TextNode: - // Nothing to do. +// EscapeSet rewrites the template set to guarantee that the output of any of +// the named templates is properly escaped. +// Names should include the names of all templates that might be Executed but +// need not include helper templates. +// If no error is returned, then the named templates have been modified. +// Otherwise the named templates have been rendered unusable. +func EscapeSet(s *template.Set, names ...string) (*template.Set, os.Error) { + if len(names) == 0 { + // TODO: Maybe add a method to Set to enumerate template names + // and use those instead. + return nil, &Error{ErrNoNames, "", 0, "must specify names of top level templates"} + } + e := newEscaper(s) + for _, name := range names { + c, _ := e.escapeTree(context{}, name, 0) + var err os.Error + if c.err != nil { + err, c.err.Name = c.err, name + } else if c.state != stateText { + err = &Error{ErrEndContext, name, 0, fmt.Sprintf("ends in a non-text context: %v", c)} + } + if err != nil { + // Prevent execution of unsafe templates. + for _, name := range names { + if t := s.Template(name); t != nil { + t.Tree = nil + } + } + return nil, err + } + } + e.commit() + return s, nil +} + +// funcMap maps command names to functions that render their inputs safe. +var funcMap = template.FuncMap{ + "exp_template_html_attrescaper": attrEscaper, + "exp_template_html_commentescaper": commentEscaper, + "exp_template_html_cssescaper": cssEscaper, + "exp_template_html_cssvaluefilter": cssValueFilter, + "exp_template_html_htmlnamefilter": htmlNameFilter, + "exp_template_html_htmlescaper": htmlEscaper, + "exp_template_html_jsregexpescaper": jsRegexpEscaper, + "exp_template_html_jsstrescaper": jsStrEscaper, + "exp_template_html_jsvalescaper": jsValEscaper, + "exp_template_html_nospaceescaper": htmlNospaceEscaper, + "exp_template_html_rcdataescaper": rcdataEscaper, + "exp_template_html_urlescaper": urlEscaper, + "exp_template_html_urlfilter": urlFilter, + "exp_template_html_urlnormalizer": urlNormalizer, +} + +// equivEscapers matches contextual escapers to equivalent template builtins. +var equivEscapers = map[string]string{ + "exp_template_html_attrescaper": "html", + "exp_template_html_htmlescaper": "html", + "exp_template_html_nospaceescaper": "html", + "exp_template_html_rcdataescaper": "html", + "exp_template_html_urlescaper": "urlquery", + "exp_template_html_urlnormalizer": "urlquery", +} + +// escaper collects type inferences about templates and changes needed to make +// templates injection safe. +type escaper struct { + // set is the template set being escaped. + set *template.Set + // output[templateName] is the output context for a templateName that + // has been mangled to include its input context. + output map[string]context + // derived[c.mangle(name)] maps to a template derived from the template + // named name templateName for the start context c. + derived map[string]*template.Template + // called[templateName] is a set of called mangled template names. + called map[string]bool + // xxxNodeEdits are the accumulated edits to apply during commit. + // Such edits are not applied immediately in case a template set + // executes a given template in different escaping contexts. + actionNodeEdits map[*parse.ActionNode][]string + templateNodeEdits map[*parse.TemplateNode]string + textNodeEdits map[*parse.TextNode][]byte +} + +// newEscaper creates a blank escaper for the given set. +func newEscaper(s *template.Set) *escaper { + return &escaper{ + s, + map[string]context{}, + map[string]*template.Template{}, + map[string]bool{}, + map[*parse.ActionNode][]string{}, + map[*parse.TemplateNode]string{}, + map[*parse.TextNode][]byte{}, + } +} + +// filterFailsafe is an innocuous word that is emitted in place of unsafe values +// by sanitizer functions. It is not a keyword in any programming language, +// contains no special characters, is not empty, and when it appears in output +// it is distinct enough that a developer can find the source of the problem +// via a search engine. +const filterFailsafe = "ZgotmplZ" + +// escape escapes a template node. +func (e *escaper) escape(c context, n parse.Node) context { + switch n := n.(type) { case *parse.ActionNode: - escapeActionNode(n) + return e.escapeAction(c, n) case *parse.IfNode: - escapeIfNode(n) + return e.escapeBranch(c, &n.BranchNode, "if") + case *parse.ListNode: + return e.escapeList(c, n) case *parse.RangeNode: - escapeRangeNode(n) + return e.escapeBranch(c, &n.BranchNode, "range") case *parse.TemplateNode: - // Nothing to do. + return e.escapeTemplate(c, n) + case *parse.TextNode: + return e.escapeText(c, n) case *parse.WithNode: - escapeWithNode(n) + return e.escapeBranch(c, &n.BranchNode, "with") + } + panic("escaping " + n.String() + " is unimplemented") +} + +// escapeAction escapes an action template node. +func (e *escaper) escapeAction(c context, n *parse.ActionNode) context { + if len(n.Pipe.Decl) != 0 { + // A local variable assignment, not an interpolation. + return c + } + c = nudge(c) + s := make([]string, 0, 3) + switch c.state { + case stateError: + return c + case stateURL, stateCSSDqStr, stateCSSSqStr, stateCSSDqURL, stateCSSSqURL, stateCSSURL: + switch c.urlPart { + case urlPartNone: + s = append(s, "exp_template_html_urlfilter") + fallthrough + case urlPartPreQuery: + switch c.state { + case stateCSSDqStr, stateCSSSqStr: + s = append(s, "exp_template_html_cssescaper") + default: + s = append(s, "exp_template_html_urlnormalizer") + } + case urlPartQueryOrFrag: + s = append(s, "exp_template_html_urlescaper") + case urlPartUnknown: + return context{ + state: stateError, + err: errorf(ErrAmbigContext, n.Line, "%s appears in an ambiguous URL context", n), + } + default: + panic(c.urlPart.String()) + } + case stateJS: + s = append(s, "exp_template_html_jsvalescaper") + // A slash after a value starts a div operator. + c.jsCtx = jsCtxDivOp + case stateJSDqStr, stateJSSqStr: + s = append(s, "exp_template_html_jsstrescaper") + case stateJSRegexp: + s = append(s, "exp_template_html_jsregexpescaper") + case stateCSS: + s = append(s, "exp_template_html_cssvaluefilter") + case stateText: + s = append(s, "exp_template_html_htmlescaper") + case stateRCDATA: + s = append(s, "exp_template_html_rcdataescaper") + case stateAttr: + // Handled below in delim check. + case stateAttrName, stateTag: + c.state = stateAttrName + s = append(s, "exp_template_html_htmlnamefilter") default: - panic("handling for " + node.String() + " not implemented") - // TODO: Handle other inner node types. + if isComment(c.state) { + s = append(s, "exp_template_html_commentescaper") + } else { + panic("unexpected state " + c.state.String()) + } } + switch c.delim { + case delimNone: + // No extra-escaping needed for raw text content. + case delimSpaceOrTagEnd: + s = append(s, "exp_template_html_nospaceescaper") + default: + s = append(s, "exp_template_html_attrescaper") + } + e.editActionNode(n, s) + return c } -// escapeListNode recursively escapes its input's children. -func escapeListNode(node *parse.ListNode) { - if node == nil { +// ensurePipelineContains ensures that the pipeline has commands with +// the identifiers in s in order. +// If the pipeline already has some of the sanitizers, do not interfere. +// For example, if p is (.X | html) and s is ["escapeJSVal", "html"] then it +// has one matching, "html", and one to insert, "escapeJSVal", to produce +// (.X | escapeJSVal | html). +func ensurePipelineContains(p *parse.PipeNode, s []string) { + if len(s) == 0 { return } - children := node.Nodes - for _, child := range children { - escapeNode(child) + n := len(p.Cmds) + // Find the identifiers at the end of the command chain. + idents := p.Cmds + for i := n - 1; i >= 0; i-- { + if cmd := p.Cmds[i]; len(cmd.Args) != 0 { + if id, ok := cmd.Args[0].(*parse.IdentifierNode); ok { + if id.Ident == "noescape" { + return + } + continue + } + } + idents = p.Cmds[i+1:] + } + dups := 0 + for _, id := range idents { + if escFnsEq(s[dups], (id.Args[0].(*parse.IdentifierNode)).Ident) { + dups++ + if dups == len(s) { + return + } + } + } + newCmds := make([]*parse.CommandNode, n-len(idents), n+len(s)-dups) + copy(newCmds, p.Cmds) + // Merge existing identifier commands with the sanitizers needed. + for _, id := range idents { + i := indexOfStr((id.Args[0].(*parse.IdentifierNode)).Ident, s, escFnsEq) + if i != -1 { + for _, name := range s[:i] { + newCmds = appendCmd(newCmds, newIdentCmd(name)) + } + s = s[i+1:] + } + newCmds = appendCmd(newCmds, id) + } + // Create any remaining sanitizers. + for _, name := range s { + newCmds = appendCmd(newCmds, newIdentCmd(name)) } + p.Cmds = newCmds } -// escapeActionNode adds a pipeline call to the end that escapes the result -// of the expression before it is interpolated into the template output. -func escapeActionNode(node *parse.ActionNode) { - pipe := node.Pipe +// redundantFuncs[a][b] implies that funcMap[b](funcMap[a](x)) == funcMap[a](x) +// for all x. +var redundantFuncs = map[string]map[string]bool{ + "exp_template_html_commentescaper": { + "exp_template_html_attrescaper": true, + "exp_template_html_nospaceescaper": true, + "exp_template_html_htmlescaper": true, + }, + "exp_template_html_cssescaper": { + "exp_template_html_attrescaper": true, + }, + "exp_template_html_jsregexpescaper": { + "exp_template_html_attrescaper": true, + }, + "exp_template_html_jsstrescaper": { + "exp_template_html_attrescaper": true, + }, + "exp_template_html_urlescaper": { + "exp_template_html_urlnormalizer": true, + }, +} - cmds := pipe.Cmds - nCmds := len(cmds) +// appendCmd appends the given command to the end of the command pipeline +// unless it is redundant with the last command. +func appendCmd(cmds []*parse.CommandNode, cmd *parse.CommandNode) []*parse.CommandNode { + if n := len(cmds); n != 0 { + last, ok := cmds[n-1].Args[0].(*parse.IdentifierNode) + next, _ := cmd.Args[0].(*parse.IdentifierNode) + if ok && redundantFuncs[last.Ident][next.Ident] { + return cmds + } + } + return append(cmds, cmd) +} - // If it already has an escaping command, do not interfere. - if nCmds != 0 { - if lastCmd := cmds[nCmds-1]; len(lastCmd.Args) != 0 { - // TODO: Recognize url and js as escaping functions once - // we have enough context to know whether additional - // escaping is necessary. - if arg, ok := lastCmd.Args[0].(*parse.IdentifierNode); ok && arg.Ident == "html" { - return - } +// indexOfStr is the first i such that eq(s, strs[i]) or -1 if s was not found. +func indexOfStr(s string, strs []string, eq func(a, b string) bool) int { + for i, t := range strs { + if eq(s, t) { + return i } } + return -1 +} - htmlEscapeCommand := parse.CommandNode{ +// escFnsEq returns whether the two escaping functions are equivalent. +func escFnsEq(a, b string) bool { + if e := equivEscapers[a]; e != "" { + a = e + } + if e := equivEscapers[b]; e != "" { + b = e + } + return a == b +} + +// newIdentCmd produces a command containing a single identifier node. +func newIdentCmd(identifier string) *parse.CommandNode { + return &parse.CommandNode{ NodeType: parse.NodeCommand, - Args: []parse.Node{parse.NewIdentifier("html")}, + Args: []parse.Node{parse.NewIdentifier(identifier)}, } +} - node.Pipe.Cmds = append(node.Pipe.Cmds, &htmlEscapeCommand) +// nudge returns the context that would result from following empty string +// transitions from the input context. +// For example, parsing: +// `<a href=` +// will end in context{stateBeforeValue, attrURL}, but parsing one extra rune: +// `<a href=x` +// will end in context{stateURL, delimSpaceOrTagEnd, ...}. +// There are two transitions that happen when the 'x' is seen: +// (1) Transition from a before-value state to a start-of-value state without +// consuming any character. +// (2) Consume 'x' and transition past the first value character. +// In this case, nudging produces the context after (1) happens. +func nudge(c context) context { + switch c.state { + case stateTag: + // In `<foo {{.}}`, the action should emit an attribute. + c.state = stateAttrName + case stateBeforeValue: + // In `<foo bar={{.}}`, the action is an undelimited value. + c.state, c.delim, c.attr = attrStartStates[c.attr], delimSpaceOrTagEnd, attrNone + case stateAfterName: + // In `<foo bar {{.}}`, the action is an attribute name. + c.state, c.attr = stateAttrName, attrNone + } + return c } -// escapeIfNode recursively escapes the if and then clauses but leaves the -// condition unchanged. -func escapeIfNode(node *parse.IfNode) { - escapeListNode(node.List) - escapeListNode(node.ElseList) +// join joins the two contexts of a branch template node. The result is an +// error context if either of the input contexts are error contexts, or if the +// the input contexts differ. +func join(a, b context, line int, nodeName string) context { + if a.state == stateError { + return a + } + if b.state == stateError { + return b + } + if a.eq(b) { + return a + } + + c := a + c.urlPart = b.urlPart + if c.eq(b) { + // The contexts differ only by urlPart. + c.urlPart = urlPartUnknown + return c + } + + c = a + c.jsCtx = b.jsCtx + if c.eq(b) { + // The contexts differ only by jsCtx. + c.jsCtx = jsCtxUnknown + return c + } + + // Allow a nudged context to join with an unnudged one. + // This means that + // <p title={{if .C}}{{.}}{{end}} + // ends in an unquoted value state even though the else branch + // ends in stateBeforeValue. + if c, d := nudge(a), nudge(b); !(c.eq(a) && d.eq(b)) { + if e := join(c, d, line, nodeName); e.state != stateError { + return e + } + } + + return context{ + state: stateError, + err: errorf(ErrBranchEnd, line, "{{%s}} branches end in different contexts: %v, %v", nodeName, a, b), + } } -// escapeRangeNode recursively escapes the loop body and else clause but -// leaves the series unchanged. -func escapeRangeNode(node *parse.RangeNode) { - escapeListNode(node.List) - escapeListNode(node.ElseList) +// escapeBranch escapes a branch template node: "if", "range" and "with". +func (e *escaper) escapeBranch(c context, n *parse.BranchNode, nodeName string) context { + c0 := e.escapeList(c, n.List) + if nodeName == "range" && c0.state != stateError { + // The "true" branch of a "range" node can execute multiple times. + // We check that executing n.List once results in the same context + // as executing n.List twice. + c1, _ := e.escapeListConditionally(c0, n.List, nil) + c0 = join(c0, c1, n.Line, nodeName) + if c0.state == stateError { + // Make clear that this is a problem on loop re-entry + // since developers tend to overlook that branch when + // debugging templates. + c0.err.Line = n.Line + c0.err.Description = "on range loop re-entry: " + c0.err.Description + return c0 + } + } + c1 := e.escapeList(c, n.ElseList) + return join(c0, c1, n.Line, nodeName) } -// escapeWithNode recursively escapes the scope body and else clause but -// leaves the pipeline unchanged. -func escapeWithNode(node *parse.WithNode) { - escapeListNode(node.List) - escapeListNode(node.ElseList) +// escapeList escapes a list template node. +func (e *escaper) escapeList(c context, n *parse.ListNode) context { + if n == nil { + return c + } + for _, m := range n.Nodes { + c = e.escape(c, m) + } + return c +} + +// escapeListConditionally escapes a list node but only preserves edits and +// inferences in e if the inferences and output context satisfy filter. +// It returns the best guess at an output context, and the result of the filter +// which is the same as whether e was updated. +func (e *escaper) escapeListConditionally(c context, n *parse.ListNode, filter func(*escaper, context) bool) (context, bool) { + e1 := newEscaper(e.set) + // Make type inferences available to f. + for k, v := range e.output { + e1.output[k] = v + } + c = e1.escapeList(c, n) + ok := filter != nil && filter(e1, c) + if ok { + // Copy inferences and edits from e1 back into e. + for k, v := range e1.output { + e.output[k] = v + } + for k, v := range e1.derived { + e.derived[k] = v + } + for k, v := range e1.called { + e.called[k] = v + } + for k, v := range e1.actionNodeEdits { + e.editActionNode(k, v) + } + for k, v := range e1.templateNodeEdits { + e.editTemplateNode(k, v) + } + for k, v := range e1.textNodeEdits { + e.editTextNode(k, v) + } + } + return c, ok +} + +// escapeTemplate escapes a {{template}} call node. +func (e *escaper) escapeTemplate(c context, n *parse.TemplateNode) context { + c, name := e.escapeTree(c, n.Name, n.Line) + if name != n.Name { + e.editTemplateNode(n, name) + } + return c +} + +// escapeTree escapes the named template starting in the given context as +// necessary and returns its output context. +func (e *escaper) escapeTree(c context, name string, line int) (context, string) { + // Mangle the template name with the input context to produce a reliable + // identifier. + dname := c.mangle(name) + e.called[dname] = true + if out, ok := e.output[dname]; ok { + // Already escaped. + return out, dname + } + t := e.template(name) + if t == nil { + return context{ + state: stateError, + err: errorf(ErrNoSuchTemplate, line, "no such template %s", name), + }, dname + } + if dname != name { + // Use any template derived during an earlier call to EscapeSet + // with different top level templates, or clone if necessary. + dt := e.template(dname) + if dt == nil { + dt = template.New(dname) + dt.Tree = &parse.Tree{Name: dname, Root: cloneList(t.Root)} + e.derived[dname] = dt + } + t = dt + } + return e.computeOutCtx(c, t), dname +} + +// computeOutCtx takes a template and its start context and computes the output +// context while storing any inferences in e. +func (e *escaper) computeOutCtx(c context, t *template.Template) context { + // Propagate context over the body. + c1, ok := e.escapeTemplateBody(c, t) + if !ok { + // Look for a fixed point by assuming c1 as the output context. + if c2, ok2 := e.escapeTemplateBody(c1, t); ok2 { + c1, ok = c2, true + } + // Use c1 as the error context if neither assumption worked. + } + if !ok && c1.state != stateError { + return context{ + state: stateError, + // TODO: Find the first node with a line in t.Tree.Root + err: errorf(ErrOutputContext, 0, "cannot compute output context for template %s", t.Name()), + } + } + return c1 +} + +// escapeTemplateBody escapes the given template assuming the given output +// context, and returns the best guess at the output context and whether the +// assumption was correct. +func (e *escaper) escapeTemplateBody(c context, t *template.Template) (context, bool) { + filter := func(e1 *escaper, c1 context) bool { + if c1.state == stateError { + // Do not update the input escaper, e. + return false + } + if !e1.called[t.Name()] { + // If t is not recursively called, then c1 is an + // accurate output context. + return true + } + // c1 is accurate if it matches our assumed output context. + return c.eq(c1) + } + // We need to assume an output context so that recursive template calls + // take the fast path out of escapeTree instead of infinitely recursing. + // Naively assuming that the input context is the same as the output + // works >90% of the time. + e.output[t.Name()] = c + return e.escapeListConditionally(c, t.Tree.Root, filter) +} + +// delimEnds maps each delim to a string of characters that terminate it. +var delimEnds = [...]string{ + delimDoubleQuote: `"`, + delimSingleQuote: "'", + // Determined empirically by running the below in various browsers. + // var div = document.createElement("DIV"); + // for (var i = 0; i < 0x10000; ++i) { + // div.innerHTML = "<span title=x" + String.fromCharCode(i) + "-bar>"; + // if (div.getElementsByTagName("SPAN")[0].title.indexOf("bar") < 0) + // document.write("<p>U+" + i.toString(16)); + // } + delimSpaceOrTagEnd: " \t\n\f\r>", +} + +var doctypeBytes = []byte("<!DOCTYPE") + +// escapeText escapes a text template node. +func (e *escaper) escapeText(c context, n *parse.TextNode) context { + s, written, i, b := n.Text, 0, 0, new(bytes.Buffer) + for i != len(s) { + c1, nread := contextAfterText(c, s[i:]) + i1 := i + nread + if c.state == stateText || c.state == stateRCDATA { + end := i1 + if c1.state != c.state { + for j := end - 1; j >= i; j-- { + if s[j] == '<' { + end = j + break + } + } + } + for j := i; j < end; j++ { + if s[j] == '<' && !bytes.HasPrefix(s[j:], doctypeBytes) { + b.Write(s[written:j]) + b.WriteString("<") + written = j + 1 + } + } + } else if isComment(c.state) && c.delim == delimNone { + switch c.state { + case stateJSBlockCmt: + // http://es5.github.com/#x7.4: + // "Comments behave like white space and are + // discarded except that, if a MultiLineComment + // contains a line terminator character, then + // the entire comment is considered to be a + // LineTerminator for purposes of parsing by + // the syntactic grammar." + if bytes.IndexAny(s[written:i1], "\n\r\u2028\u2029") != -1 { + b.WriteByte('\n') + } else { + b.WriteByte(' ') + } + case stateCSSBlockCmt: + b.WriteByte(' ') + } + written = i1 + } + if c.state != c1.state && isComment(c1.state) && c1.delim == delimNone { + // Preserve the portion between written and the comment start. + cs := i1 - 2 + if c1.state == stateHTMLCmt { + // "<!--" instead of "/*" or "//" + cs -= 2 + } + b.Write(s[written:cs]) + written = i1 + } + if i == i1 && c.state == c1.state { + panic(fmt.Sprintf("infinite loop from %v to %v on %q..%q", c, c1, s[:i], s[i:])) + } + c, i = c1, i1 + } + + if written != 0 && c.state != stateError { + if !isComment(c.state) || c.delim != delimNone { + b.Write(n.Text[written:]) + } + e.editTextNode(n, b.Bytes()) + } + return c +} + +// contextAfterText starts in context c, consumes some tokens from the front of +// s, then returns the context after those tokens and the unprocessed suffix. +func contextAfterText(c context, s []byte) (context, int) { + if c.delim == delimNone { + c1, i := tSpecialTagEnd(c, s) + if i == 0 { + // A special end tag (`</script>`) has been seen and + // all content preceding it has been consumed. + return c1, 0 + } + // Consider all content up to any end tag. + return transitionFunc[c.state](c, s[:i]) + } + + i := bytes.IndexAny(s, delimEnds[c.delim]) + if i == -1 { + i = len(s) + } + if c.delim == delimSpaceOrTagEnd { + // http://www.w3.org/TR/html5/tokenization.html#attribute-value-unquoted-state + // lists the runes below as error characters. + // Error out because HTML parsers may differ on whether + // "<a id= onclick=f(" ends inside id's or onchange's value, + // "<a class=`foo " ends inside a value, + // "<a style=font:'Arial'" needs open-quote fixup. + // IE treats '`' as a quotation character. + if j := bytes.IndexAny(s[:i], "\"'<=`"); j >= 0 { + return context{ + state: stateError, + err: errorf(ErrBadHTML, 0, "%q in unquoted attr: %q", s[j:j+1], s[:i]), + }, len(s) + } + } + if i == len(s) { + // Remain inside the attribute. + // Decode the value so non-HTML rules can easily handle + // <button onclick="alert("Hi!")"> + // without having to entity decode token boundaries. + for u := []byte(html.UnescapeString(string(s))); len(u) != 0; { + c1, i1 := transitionFunc[c.state](c, u) + c, u = c1, u[i1:] + } + return c, len(s) + } + if c.delim != delimSpaceOrTagEnd { + // Consume any quote. + i++ + } + // On exiting an attribute, we discard all state information + // except the state and element. + return context{state: stateTag, element: c.element}, i +} + +// editActionNode records a change to an action pipeline for later commit. +func (e *escaper) editActionNode(n *parse.ActionNode, cmds []string) { + if _, ok := e.actionNodeEdits[n]; ok { + panic(fmt.Sprintf("node %s shared between templates", n)) + } + e.actionNodeEdits[n] = cmds +} + +// editTemplateNode records a change to a {{template}} callee for later commit. +func (e *escaper) editTemplateNode(n *parse.TemplateNode, callee string) { + if _, ok := e.templateNodeEdits[n]; ok { + panic(fmt.Sprintf("node %s shared between templates", n)) + } + e.templateNodeEdits[n] = callee +} + +// editTextNode records a change to a text node for later commit. +func (e *escaper) editTextNode(n *parse.TextNode, text []byte) { + if _, ok := e.textNodeEdits[n]; ok { + panic(fmt.Sprintf("node %s shared between templates", n)) + } + e.textNodeEdits[n] = text +} + +// commit applies changes to actions and template calls needed to contextually +// autoescape content and adds any derived templates to the set. +func (e *escaper) commit() { + for name, _ := range e.output { + e.template(name).Funcs(funcMap) + } + for _, t := range e.derived { + e.set.Add(t) + } + for n, s := range e.actionNodeEdits { + ensurePipelineContains(n.Pipe, s) + } + for n, name := range e.templateNodeEdits { + n.Name = name + } + for n, s := range e.textNodeEdits { + n.Text = s + } +} + +// template returns the named template given a mangled template name. +func (e *escaper) template(name string) *template.Template { + t := e.set.Template(name) + if t == nil { + t = e.derived[name] + } + return t } diff --git a/libgo/go/exp/template/html/escape_test.go b/libgo/go/exp/template/html/escape_test.go index 345a752..a4ea759 100644 --- a/libgo/go/exp/template/html/escape_test.go +++ b/libgo/go/exp/template/html/escape_test.go @@ -6,70 +6,1618 @@ package html import ( "bytes" + "fmt" + "json" + "os" + "strings" "template" + "template/parse" "testing" ) -type data struct { - F, T bool - C, G, H string - A, E []string +type badMarshaler struct{} + +func (x *badMarshaler) MarshalJSON() ([]byte, os.Error) { + // Keys in valid JSON must be double quoted as must all strings. + return []byte("{ foo: 'not quite valid JSON' }"), nil } -var testData = data{ - F: false, - T: true, - C: "<Cincinatti>", - G: "<Goodbye>", - H: "<Hello>", - A: []string{"<a>", "<b>"}, - E: []string{}, +type goodMarshaler struct{} + +func (x *goodMarshaler) MarshalJSON() ([]byte, os.Error) { + return []byte(`{ "<foo>": "O'Reilly" }`), nil } -type testCase struct { - name string - input string - output string +func TestEscape(t *testing.T) { + var data = struct { + F, T bool + C, G, H string + A, E []string + B, M json.Marshaler + N int + Z *int + W HTML + }{ + F: false, + T: true, + C: "<Cincinatti>", + G: "<Goodbye>", + H: "<Hello>", + A: []string{"<a>", "<b>"}, + E: []string{}, + N: 42, + B: &badMarshaler{}, + M: &goodMarshaler{}, + Z: nil, + W: HTML(`¡<b class="foo">Hello</b>, <textarea>O'World</textarea>!`), + } + + tests := []struct { + name string + input string + output string + }{ + { + "if", + "{{if .T}}Hello{{end}}, {{.C}}!", + "Hello, <Cincinatti>!", + }, + { + "else", + "{{if .F}}{{.H}}{{else}}{{.G}}{{end}}!", + "<Goodbye>!", + }, + { + "overescaping1", + "Hello, {{.C | html}}!", + "Hello, <Cincinatti>!", + }, + { + "overescaping2", + "Hello, {{html .C}}!", + "Hello, <Cincinatti>!", + }, + { + "overescaping3", + "{{with .C}}{{$msg := .}}Hello, {{$msg}}!{{end}}", + "Hello, <Cincinatti>!", + }, + { + "assignment", + "{{if $x := .H}}{{$x}}{{end}}", + "<Hello>", + }, + { + "withBody", + "{{with .H}}{{.}}{{end}}", + "<Hello>", + }, + { + "withElse", + "{{with .E}}{{.}}{{else}}{{.H}}{{end}}", + "<Hello>", + }, + { + "rangeBody", + "{{range .A}}{{.}}{{end}}", + "<a><b>", + }, + { + "rangeElse", + "{{range .E}}{{.}}{{else}}{{.H}}{{end}}", + "<Hello>", + }, + { + "nonStringValue", + "{{.T}}", + "true", + }, + { + "constant", + `<a href="/search?q={{"'a<b'"}}">`, + `<a href="/search?q=%27a%3cb%27">`, + }, + { + "multipleAttrs", + "<a b=1 c={{.H}}>", + "<a b=1 c=<Hello>>", + }, + { + "urlStartRel", + `<a href='{{"/foo/bar?a=b&c=d"}}'>`, + `<a href='/foo/bar?a=b&c=d'>`, + }, + { + "urlStartAbsOk", + `<a href='{{"http://example.com/foo/bar?a=b&c=d"}}'>`, + `<a href='http://example.com/foo/bar?a=b&c=d'>`, + }, + { + "protocolRelativeURLStart", + `<a href='{{"//example.com:8000/foo/bar?a=b&c=d"}}'>`, + `<a href='//example.com:8000/foo/bar?a=b&c=d'>`, + }, + { + "pathRelativeURLStart", + `<a href="{{"/javascript:80/foo/bar"}}">`, + `<a href="/javascript:80/foo/bar">`, + }, + { + "dangerousURLStart", + `<a href='{{"javascript:alert(%22pwned%22)"}}'>`, + `<a href='#ZgotmplZ'>`, + }, + { + "dangerousURLStart2", + `<a href=' {{"javascript:alert(%22pwned%22)"}}'>`, + `<a href=' #ZgotmplZ'>`, + }, + { + "nonHierURL", + `<a href={{"mailto:Muhammed \"The Greatest\" Ali <m.ali@example.com>"}}>`, + `<a href=mailto:Muhammed%20%22The%20Greatest%22%20Ali%20%3cm.ali@example.com%3e>`, + }, + { + "urlPath", + `<a href='http://{{"javascript:80"}}/foo'>`, + `<a href='http://javascript:80/foo'>`, + }, + { + "urlQuery", + `<a href='/search?q={{.H}}'>`, + `<a href='/search?q=%3cHello%3e'>`, + }, + { + "urlFragment", + `<a href='/faq#{{.H}}'>`, + `<a href='/faq#%3cHello%3e'>`, + }, + { + "urlBranch", + `<a href="{{if .F}}/foo?a=b{{else}}/bar{{end}}">`, + `<a href="/bar">`, + }, + { + "urlBranchConflictMoot", + `<a href="{{if .T}}/foo?a={{else}}/bar#{{end}}{{.C}}">`, + `<a href="/foo?a=%3cCincinatti%3e">`, + }, + { + "jsStrValue", + "<button onclick='alert({{.H}})'>", + `<button onclick='alert("\u003cHello\u003e")'>`, + }, + { + "jsNumericValue", + "<button onclick='alert({{.N}})'>", + `<button onclick='alert( 42 )'>`, + }, + { + "jsBoolValue", + "<button onclick='alert({{.T}})'>", + `<button onclick='alert( true )'>`, + }, + { + "jsNilValue", + "<button onclick='alert(typeof{{.Z}})'>", + `<button onclick='alert(typeof null )'>`, + }, + { + "jsObjValue", + "<button onclick='alert({{.A}})'>", + `<button onclick='alert(["\u003ca\u003e","\u003cb\u003e"])'>`, + }, + { + "jsObjValueScript", + "<script>alert({{.A}})</script>", + `<script>alert(["\u003ca\u003e","\u003cb\u003e"])</script>`, + }, + { + "jsObjValueNotOverEscaped", + "<button onclick='alert({{.A | html}})'>", + `<button onclick='alert(["\u003ca\u003e","\u003cb\u003e"])'>`, + }, + { + "jsStr", + "<button onclick='alert("{{.H}}")'>", + `<button onclick='alert("\x3cHello\x3e")'>`, + }, + { + "badMarshaller", + `<button onclick='alert(1/{{.B}}in numbers)'>`, + `<button onclick='alert(1/ /* json: error calling MarshalJSON for type *html.badMarshaler: invalid character 'f' looking for beginning of object key string */null in numbers)'>`, + }, + { + "jsMarshaller", + `<button onclick='alert({{.M}})'>`, + `<button onclick='alert({"<foo>":"O'Reilly"})'>`, + }, + { + "jsStrNotUnderEscaped", + "<button onclick='alert({{.C | urlquery}})'>", + // URL escaped, then quoted for JS. + `<button onclick='alert("%3CCincinatti%3E")'>`, + }, + { + "jsRe", + `<button onclick='alert(/{{"foo+bar"}}/.test(""))'>`, + `<button onclick='alert(/foo\x2bbar/.test(""))'>`, + }, + { + "jsReBlank", + `<script>alert(/{{""}}/.test(""));</script>`, + `<script>alert(/(?:)/.test(""));</script>`, + }, + { + "jsReAmbigOk", + `<script>{{if true}}var x = 1{{end}}</script>`, + // The {if} ends in an ambiguous jsCtx but there is + // no slash following so we shouldn't care. + `<script>var x = 1</script>`, + }, + { + "styleBidiKeywordPassed", + `<p style="dir: {{"ltr"}}">`, + `<p style="dir: ltr">`, + }, + { + "styleBidiPropNamePassed", + `<p style="border-{{"left"}}: 0; border-{{"right"}}: 1in">`, + `<p style="border-left: 0; border-right: 1in">`, + }, + { + "styleExpressionBlocked", + `<p style="width: {{"expression(alert(1337))"}}">`, + `<p style="width: ZgotmplZ">`, + }, + { + "styleTagSelectorPassed", + `<style>{{"p"}} { color: pink }</style>`, + `<style>p { color: pink }</style>`, + }, + { + "styleIDPassed", + `<style>p{{"#my-ID"}} { font: Arial }</style>`, + `<style>p#my-ID { font: Arial }</style>`, + }, + { + "styleClassPassed", + `<style>p{{".my_class"}} { font: Arial }</style>`, + `<style>p.my_class { font: Arial }</style>`, + }, + { + "styleQuantityPassed", + `<a style="left: {{"2em"}}; top: {{0}}">`, + `<a style="left: 2em; top: 0">`, + }, + { + "stylePctPassed", + `<table style=width:{{"100%"}}>`, + `<table style=width:100%>`, + }, + { + "styleColorPassed", + `<p style="color: {{"#8ff"}}; background: {{"#000"}}">`, + `<p style="color: #8ff; background: #000">`, + }, + { + "styleObfuscatedExpressionBlocked", + `<p style="width: {{" e\78preS\0Sio/**/n(alert(1337))"}}">`, + `<p style="width: ZgotmplZ">`, + }, + { + "styleMozBindingBlocked", + `<p style="{{"-moz-binding(alert(1337))"}}: ...">`, + `<p style="ZgotmplZ: ...">`, + }, + { + "styleObfuscatedMozBindingBlocked", + `<p style="{{" -mo\7a-B\0I/**/nding(alert(1337))"}}: ...">`, + `<p style="ZgotmplZ: ...">`, + }, + { + "styleFontNameString", + `<p style='font-family: "{{"Times New Roman"}}"'>`, + `<p style='font-family: "Times New Roman"'>`, + }, + { + "styleFontNameString", + `<p style='font-family: "{{"Times New Roman"}}", "{{"sans-serif"}}"'>`, + `<p style='font-family: "Times New Roman", "sans-serif"'>`, + }, + { + "styleFontNameUnquoted", + `<p style='font-family: {{"Times New Roman"}}'>`, + `<p style='font-family: Times New Roman'>`, + }, + { + "styleURLQueryEncoded", + `<p style="background: url(/img?name={{"O'Reilly Animal(1)<2>.png"}})">`, + `<p style="background: url(/img?name=O%27Reilly%20Animal%281%29%3c2%3e.png)">`, + }, + { + "styleQuotedURLQueryEncoded", + `<p style="background: url('/img?name={{"O'Reilly Animal(1)<2>.png"}}')">`, + `<p style="background: url('/img?name=O%27Reilly%20Animal%281%29%3c2%3e.png')">`, + }, + { + "styleStrQueryEncoded", + `<p style="background: '/img?name={{"O'Reilly Animal(1)<2>.png"}}'">`, + `<p style="background: '/img?name=O%27Reilly%20Animal%281%29%3c2%3e.png'">`, + }, + { + "styleURLBadProtocolBlocked", + `<a style="background: url('{{"javascript:alert(1337)"}}')">`, + `<a style="background: url('#ZgotmplZ')">`, + }, + { + "styleStrBadProtocolBlocked", + `<a style="background: '{{"vbscript:alert(1337)"}}'">`, + `<a style="background: '#ZgotmplZ'">`, + }, + { + "styleStrEncodedProtocolEncoded", + `<a style="background: '{{"javascript\\3a alert(1337)"}}'">`, + // The CSS string 'javascript\\3a alert(1337)' does not contains a colon. + `<a style="background: 'javascript\\3a alert\28 1337\29 '">`, + }, + { + "styleURLGoodProtocolPassed", + `<a style="background: url('{{"http://oreilly.com/O'Reilly Animals(1)<2>;{}.html"}}')">`, + `<a style="background: url('http://oreilly.com/O%27Reilly%20Animals%281%29%3c2%3e;%7b%7d.html')">`, + }, + { + "styleStrGoodProtocolPassed", + `<a style="background: '{{"http://oreilly.com/O'Reilly Animals(1)<2>;{}.html"}}'">`, + `<a style="background: 'http\3a\2f\2foreilly.com\2fO\27Reilly Animals\28 1\29\3c 2\3e\3b\7b\7d.html'">`, + }, + { + "styleURLEncodedForHTMLInAttr", + `<a style="background: url('{{"/search?img=foo&size=icon"}}')">`, + `<a style="background: url('/search?img=foo&size=icon')">`, + }, + { + "styleURLNotEncodedForHTMLInCdata", + `<style>body { background: url('{{"/search?img=foo&size=icon"}}') }</style>`, + `<style>body { background: url('/search?img=foo&size=icon') }</style>`, + }, + { + "styleURLMixedCase", + `<p style="background: URL(#{{.H}})">`, + `<p style="background: URL(#%3cHello%3e)">`, + }, + { + "stylePropertyPairPassed", + `<a style='{{"color: red"}}'>`, + `<a style='color: red'>`, + }, + { + "styleStrSpecialsEncoded", + `<a style="font-family: '{{"/**/'\";:// \\"}}', "{{"/**/'\";:// \\"}}"">`, + `<a style="font-family: '\2f**\2f\27\22\3b\3a\2f\2f \\', "\2f**\2f\27\22\3b\3a\2f\2f \\"">`, + }, + { + "styleURLSpecialsEncoded", + `<a style="border-image: url({{"/**/'\";:// \\"}}), url("{{"/**/'\";:// \\"}}"), url('{{"/**/'\";:// \\"}}'), 'http://www.example.com/?q={{"/**/'\";:// \\"}}''">`, + `<a style="border-image: url(/**/%27%22;://%20%5c), url("/**/%27%22;://%20%5c"), url('/**/%27%22;://%20%5c'), 'http://www.example.com/?q=%2f%2a%2a%2f%27%22%3b%3a%2f%2f%20%5c''">`, + }, + { + "HTML comment", + "<b>Hello, <!-- name of world -->{{.C}}</b>", + "<b>Hello, <Cincinatti></b>", + }, + { + "HTML comment not first < in text node.", + "<<!-- -->!--", + "<!--", + }, + { + "HTML normalization 1", + "a < b", + "a < b", + }, + { + "HTML normalization 2", + "a << b", + "a << b", + }, + { + "HTML normalization 3", + "a<<!-- --><!-- -->b", + "a<b", + }, + { + "HTML doctype not normalized", + "<!DOCTYPE html>Hello, World!", + "<!DOCTYPE html>Hello, World!", + }, + { + "No doctype injection", + `<!{{"DOCTYPE"}}`, + "<!DOCTYPE", + }, + { + "Split HTML comment", + "<b>Hello, <!-- name of {{if .T}}city -->{{.C}}{{else}}world -->{{.W}}{{end}}</b>", + "<b>Hello, <Cincinatti></b>", + }, + { + "JS line comment", + "<script>for (;;) { if (c()) break// foo not a label\n" + + "foo({{.T}});}</script>", + "<script>for (;;) { if (c()) break\n" + + "foo( true );}</script>", + }, + { + "JS multiline block comment", + "<script>for (;;) { if (c()) break/* foo not a label\n" + + " */foo({{.T}});}</script>", + // Newline separates break from call. If newline + // removed, then break will consume label leaving + // code invalid. + "<script>for (;;) { if (c()) break\n" + + "foo( true );}</script>", + }, + { + "JS single-line block comment", + "<script>for (;;) {\n" + + "if (c()) break/* foo a label */foo;" + + "x({{.T}});}</script>", + // Newline separates break from call. If newline + // removed, then break will consume label leaving + // code invalid. + "<script>for (;;) {\n" + + "if (c()) break foo;" + + "x( true );}</script>", + }, + { + "JS block comment flush with mathematical division", + "<script>var a/*b*//c\nd</script>", + "<script>var a /c\nd</script>", + }, + { + "JS mixed comments", + "<script>var a/*b*///c\nd</script>", + "<script>var a \nd</script>", + }, + { + "CSS comments", + "<style>p// paragraph\n" + + `{border: 1px/* color */{{"#00f"}}}</style>`, + "<style>p\n" + + "{border: 1px #00f}</style>", + }, + { + "JS attr block comment", + `<a onclick="f(""); /* alert({{.H}}) */">`, + // Attribute comment tests should pass if the comments + // are successfully elided. + `<a onclick="f(""); /* alert() */">`, + }, + { + "JS attr line comment", + `<a onclick="// alert({{.G}})">`, + `<a onclick="// alert()">`, + }, + { + "CSS attr block comment", + `<a style="/* color: {{.H}} */">`, + `<a style="/* color: */">`, + }, + { + "CSS attr line comment", + `<a style="// color: {{.G}}">`, + `<a style="// color: ">`, + }, + { + "HTML substitution commented out", + "<p><!-- {{.H}} --></p>", + "<p></p>", + }, + { + "Comment ends flush with start", + "<!--{{.}}--><script>/*{{.}}*///{{.}}\n</script><style>/*{{.}}*///{{.}}\n</style><a onclick='/*{{.}}*///{{.}}' style='/*{{.}}*///{{.}}'>", + "<script> \n</script><style> \n</style><a onclick='/**///' style='/**///'>", + }, + { + "typed HTML in text", + `{{.W}}`, + `¡<b class="foo">Hello</b>, <textarea>O'World</textarea>!`, + }, + { + "typed HTML in attribute", + `<div title="{{.W}}">`, + `<div title="¡Hello, O'World!">`, + }, + { + "typed HTML in script", + `<button onclick="alert({{.W}})">`, + `<button onclick="alert("&iexcl;\u003cb class=\"foo\"\u003eHello\u003c/b\u003e, \u003ctextarea\u003eO'World\u003c/textarea\u003e!")">`, + }, + { + "typed HTML in RCDATA", + `<textarea>{{.W}}</textarea>`, + `<textarea>¡<b class="foo">Hello</b>, <textarea>O'World</textarea>!</textarea>`, + }, + { + "range in textarea", + "<textarea>{{range .A}}{{.}}{{end}}</textarea>", + "<textarea><a><b></textarea>", + }, + { + "auditable exemption from escaping", + "{{range .A}}{{. | noescape}}{{end}}", + "<a><b>", + }, + { + "No tag injection", + `{{"10$"}}<{{"script src,evil.org/pwnd.js"}}...`, + `10$<script src,evil.org/pwnd.js...`, + }, + { + "No comment injection", + `<{{"!--"}}`, + `<!--`, + }, + { + "No RCDATA end tag injection", + `<textarea><{{"/textarea "}}...</textarea>`, + `<textarea></textarea ...</textarea>`, + }, + { + "optional attrs", + `<img class="{{"iconClass"}}"` + + `{{if .T}} id="{{"<iconId>"}}"{{end}}` + + // Double quotes inside if/else. + ` src=` + + `{{if .T}}"?{{"<iconPath>"}}"` + + `{{else}}"images/cleardot.gif"{{end}}` + + // Missing space before title, but it is not a + // part of the src attribute. + `{{if .T}}title="{{"<title>"}}"{{end}}` + + // Quotes outside if/else. + ` alt="` + + `{{if .T}}{{"<alt>"}}` + + `{{else}}{{if .F}}{{"<title>"}}{{end}}` + + `{{end}}"` + + `>`, + `<img class="iconClass" id="<iconId>" src="?%3ciconPath%3e"title="<title>" alt="<alt>">`, + }, + { + "conditional valueless attr name", + `<input{{if .T}} checked{{end}} name=n>`, + `<input checked name=n>`, + }, + { + "conditional dynamic valueless attr name 1", + `<input{{if .T}} {{"checked"}}{{end}} name=n>`, + `<input checked name=n>`, + }, + { + "conditional dynamic valueless attr name 2", + `<input {{if .T}}{{"checked"}} {{end}}name=n>`, + `<input checked name=n>`, + }, + { + "dynamic attribute name", + `<img on{{"load"}}="alert({{"loaded"}})">`, + // Treated as JS since quotes are inserted. + `<img onload="alert("loaded")">`, + }, + { + "bad dynamic attribute name 1", + // Allow checked, selected, disabled, but not JS or + // CSS attributes. + `<input {{"onchange"}}="{{"doEvil()"}}">`, + `<input ZgotmplZ="doEvil()">`, + }, + { + "bad dynamic attribute name 2", + `<div {{"sTyle"}}="{{"color: expression(alert(1337))"}}">`, + `<div ZgotmplZ="color: expression(alert(1337))">`, + }, + { + "bad dynamic attribute name 3", + // Allow title or alt, but not a URL. + `<img {{"src"}}="{{"javascript:doEvil()"}}">`, + `<img ZgotmplZ="javascript:doEvil()">`, + }, + { + "bad dynamic attribute name 4", + // Structure preservation requires values to associate + // with a consistent attribute. + `<input checked {{""}}="Whose value am I?">`, + `<input checked ZgotmplZ="Whose value am I?">`, + }, + { + "dynamic element name", + `<h{{3}}><table><t{{"head"}}>...</h{{3}}>`, + `<h3><table><thead>...</h3>`, + }, + { + "bad dynamic element name", + // Dynamic element names are typically used to switch + // between (thead, tfoot, tbody), (ul, ol), (th, td), + // and other replaceable sets. + // We do not currently easily support (ul, ol). + // If we do change to support that, this test should + // catch failures to filter out special tag names which + // would violate the structure preservation property -- + // if any special tag name could be substituted, then + // the content could be raw text/RCDATA for some inputs + // and regular HTML content for others. + `<{{"script"}}>{{"doEvil()"}}</{{"script"}}>`, + `<script>doEvil()</script>`, + }, + } + + for _, test := range tests { + tmpl := template.New(test.name) + // TODO: Move noescape into template/func.go + tmpl.Funcs(template.FuncMap{ + "noescape": func(a ...interface{}) string { + return fmt.Sprint(a...) + }, + }) + tmpl = template.Must(Escape(template.Must(tmpl.Parse(test.input)))) + b := new(bytes.Buffer) + if err := tmpl.Execute(b, data); err != nil { + t.Errorf("%s: template execution failed: %s", test.name, err) + continue + } + if w, g := test.output, b.String(); w != g { + t.Errorf("%s: escaped output: want\n\t%q\ngot\n\t%q", test.name, w, g) + continue + } + } } -var testCases = []testCase{ - {"if", "{{if .T}}Hello{{end}}, {{.C}}!", "Hello, <Cincinatti>!"}, - {"else", "{{if .F}}{{.H}}{{else}}{{.G}}{{end}}!", "<Goodbye>!"}, - {"overescaping", "Hello, {{.C | html}}!", "Hello, <Cincinatti>!"}, - {"assignment", "{{if $x := .H}}{{$x}}{{end}}", "<Hello>"}, - {"withBody", "{{with .H}}{{.}}{{end}}", "<Hello>"}, - {"withElse", "{{with .E}}{{.}}{{else}}{{.H}}{{end}}", "<Hello>"}, - {"rangeBody", "{{range .A}}{{.}}{{end}}", "<a><b>"}, - {"rangeElse", "{{range .E}}{{.}}{{else}}{{.H}}{{end}}", "<Hello>"}, - {"nonStringValue", "{{.T}}", "true"}, - {"constant", `<a href="{{"'str'"}}">`, `<a href="'str'">`}, +func TestEscapeSet(t *testing.T) { + type dataItem struct { + Children []*dataItem + X string + } + + data := dataItem{ + Children: []*dataItem{ + &dataItem{X: "foo"}, + &dataItem{X: "<bar>"}, + &dataItem{ + Children: []*dataItem{ + &dataItem{X: "baz"}, + }, + }, + }, + } + + tests := []struct { + inputs map[string]string + want string + }{ + // The trivial set. + { + map[string]string{ + "main": ``, + }, + ``, + }, + // A template called in the start context. + { + map[string]string{ + "main": `Hello, {{template "helper"}}!`, + // Not a valid top level HTML template. + // "<b" is not a full tag. + "helper": `{{"<World>"}}`, + }, + `Hello, <World>!`, + }, + // A template called in a context other than the start. + { + map[string]string{ + "main": `<a onclick='a = {{template "helper"}};'>`, + // Not a valid top level HTML template. + // "<b" is not a full tag. + "helper": `{{"<a>"}}<b`, + }, + `<a onclick='a = "\u003ca\u003e"<b;'>`, + }, + // A recursive template that ends in its start context. + { + map[string]string{ + "main": `{{range .Children}}{{template "main" .}}{{else}}{{.X}} {{end}}`, + }, + `foo <bar> baz `, + }, + // A recursive helper template that ends in its start context. + { + map[string]string{ + "main": `{{template "helper" .}}`, + "helper": `{{if .Children}}<ul>{{range .Children}}<li>{{template "main" .}}</li>{{end}}</ul>{{else}}{{.X}}{{end}}`, + }, + `<ul><li>foo</li><li><bar></li><li><ul><li>baz</li></ul></li></ul>`, + }, + // Co-recursive templates that end in its start context. + { + map[string]string{ + "main": `<blockquote>{{range .Children}}{{template "helper" .}}{{end}}</blockquote>`, + "helper": `{{if .Children}}{{template "main" .}}{{else}}{{.X}}<br>{{end}}`, + }, + `<blockquote>foo<br><bar><br><blockquote>baz<br></blockquote></blockquote>`, + }, + // A template that is called in two different contexts. + { + map[string]string{ + "main": `<button onclick="title='{{template "helper"}}'; ...">{{template "helper"}}</button>`, + "helper": `{{11}} of {{"<100>"}}`, + }, + `<button onclick="title='11 of \x3c100\x3e'; ...">11 of <100></button>`, + }, + // A non-recursive template that ends in a different context. + // helper starts in jsCtxRegexp and ends in jsCtxDivOp. + { + map[string]string{ + "main": `<script>var x={{template "helper"}}/{{"42"}};</script>`, + "helper": "{{126}}", + }, + `<script>var x= 126 /"42";</script>`, + }, + // A recursive template that ends in a similar context. + { + map[string]string{ + "main": `<script>var x=[{{template "countdown" 4}}];</script>`, + "countdown": `{{.}}{{if .}},{{template "countdown" . | pred}}{{end}}`, + }, + `<script>var x=[ 4 , 3 , 2 , 1 , 0 ];</script>`, + }, + // A recursive template that ends in a different context. + /* + { + map[string]string{ + "main": `<a href="/foo{{template "helper" .}}">`, + "helper": `{{if .Children}}{{range .Children}}{{template "helper" .}}{{end}}{{else}}?x={{.X}}{{end}}`, + }, + `<a href="/foo?x=foo?x=%3cbar%3e?x=baz">`, + }, + */ + } + + // pred is a template function that returns the predecessor of a + // natural number for testing recursive templates. + fns := template.FuncMap{"pred": func(a ...interface{}) (interface{}, os.Error) { + if len(a) == 1 { + if i, _ := a[0].(int); i > 0 { + return i - 1, nil + } + } + return nil, fmt.Errorf("undefined pred(%v)", a) + }} + + for _, test := range tests { + var s template.Set + for name, src := range test.inputs { + t := template.New(name) + t.Funcs(fns) + s.Add(template.Must(t.Parse(src))) + } + s.Funcs(fns) + if _, err := EscapeSet(&s, "main"); err != nil { + t.Errorf("%s for input:\n%v", err, test.inputs) + continue + } + var b bytes.Buffer + + if err := s.Execute(&b, "main", data); err != nil { + t.Errorf("%q executing %v", err.String(), s.Template("main")) + continue + } + if got := b.String(); test.want != got { + t.Errorf("want\n\t%q\ngot\n\t%q", test.want, got) + } + } + } -func TestAutoesc(t *testing.T) { - for _, testCase := range testCases { - name := testCase.name - tmpl := template.New(name) - tmpl, err := tmpl.Parse(testCase.input) +func TestErrors(t *testing.T) { + tests := []struct { + input string + err string + }{ + // Non-error cases. + { + "{{if .Cond}}<a>{{else}}<b>{{end}}", + "", + }, + { + "{{if .Cond}}<a>{{end}}", + "", + }, + { + "{{if .Cond}}{{else}}<b>{{end}}", + "", + }, + { + "{{with .Cond}}<div>{{end}}", + "", + }, + { + "{{range .Items}}<a>{{end}}", + "", + }, + { + "<a href='/foo?{{range .Items}}&{{.K}}={{.V}}{{end}}'>", + "", + }, + // Error cases. + { + "{{if .Cond}}<a{{end}}", + "z:1: {{if}} branches", + }, + { + "{{if .Cond}}\n{{else}}\n<a{{end}}", + "z:1: {{if}} branches", + }, + { + // Missing quote in the else branch. + `{{if .Cond}}<a href="foo">{{else}}<a href="bar>{{end}}`, + "z:1: {{if}} branches", + }, + { + // Different kind of attribute: href implies a URL. + "<a {{if .Cond}}href='{{else}}title='{{end}}{{.X}}'>", + "z:1: {{if}} branches", + }, + { + "\n{{with .X}}<a{{end}}", + "z:2: {{with}} branches", + }, + { + "\n{{with .X}}<a>{{else}}<a{{end}}", + "z:2: {{with}} branches", + }, + { + "{{range .Items}}<a{{end}}", + `z:1: on range loop re-entry: "<" in attribute name: "<a"`, + }, + { + "\n{{range .Items}} x='<a{{end}}", + "z:2: on range loop re-entry: {{range}} branches", + }, + { + "<a b=1 c={{.H}}", + "z: ends in a non-text context: {stateAttr delimSpaceOrTagEnd", + }, + { + "<script>foo();", + "z: ends in a non-text context: {stateJS", + }, + { + `<a href="{{if .F}}/foo?a={{else}}/bar/{{end}}{{.H}}">`, + "z:1: (action: [(command: [F=[H]])]) appears in an ambiguous URL context", + }, + { + `<a onclick="alert('Hello \`, + `unfinished escape sequence in JS string: "Hello \\"`, + }, + { + `<a onclick='alert("Hello\, World\`, + `unfinished escape sequence in JS string: "Hello\\, World\\"`, + }, + { + `<a onclick='alert(/x+\`, + `unfinished escape sequence in JS string: "x+\\"`, + }, + { + `<a onclick="/foo[\]/`, + `unfinished JS regexp charset: "foo[\\]/"`, + }, + { + // It is ambiguous whether 1.5 should be 1\.5 or 1.5. + // Either `var x = 1/- 1.5 /i.test(x)` + // where `i.test(x)` is a method call of reference i, + // or `/-1\.5/i.test(x)` which is a method call on a + // case insensitive regular expression. + `<script>{{if false}}var x = 1{{end}}/-{{"1.5"}}/i.test(x)</script>`, + `'/' could start a division or regexp: "/-"`, + }, + { + `{{template "foo"}}`, + "z:1: no such template foo", + }, + { + `{{define "z"}}<div{{template "y"}}>{{end}}` + + // Illegal starting in stateTag but not in stateText. + `{{define "y"}} foo<b{{end}}`, + `"<" in attribute name: " foo<b"`, + }, + { + `{{define "z"}}<script>reverseList = [{{template "t"}}]</script>{{end}}` + + // Missing " after recursive call. + `{{define "t"}}{{if .Tail}}{{template "t" .Tail}}{{end}}{{.Head}}",{{end}}`, + `: cannot compute output context for template t$htmltemplate_stateJS_elementScript`, + }, + { + `<input type=button value=onclick=>`, + `exp/template/html:z: "=" in unquoted attr: "onclick="`, + }, + { + `<input type=button value= onclick=>`, + `exp/template/html:z: "=" in unquoted attr: "onclick="`, + }, + { + `<input type=button value= 1+1=2>`, + `exp/template/html:z: "=" in unquoted attr: "1+1=2"`, + }, + { + "<a class=`foo>", + "exp/template/html:z: \"`\" in unquoted attr: \"`foo\"", + }, + { + `<a style=font:'Arial'>`, + `exp/template/html:z: "'" in unquoted attr: "font:'Arial'"`, + }, + { + `<a=foo>`, + `: expected space, attr name, or end of tag, but got "=foo>"`, + }, + } + + for _, test := range tests { + var err os.Error + if strings.HasPrefix(test.input, "{{define") { + var s template.Set + _, err = s.Parse(test.input) + if err != nil { + t.Errorf("Failed to parse %q: %s", test.input, err) + continue + } + _, err = EscapeSet(&s, "z") + } else { + tmpl := template.Must(template.New("z").Parse(test.input)) + _, err = Escape(tmpl) + } + var got string if err != nil { - t.Errorf("%s: failed to parse template: %s", name, err) + got = err.String() + } + if test.err == "" { + if got != "" { + t.Errorf("input=%q: unexpected error %q", test.input, got) + } + continue + } + if strings.Index(got, test.err) == -1 { + t.Errorf("input=%q: error\n\t%q\ndoes not contain expected string\n\t%q", test.input, got, test.err) continue } + } +} - Escape(tmpl) +func TestEscapeText(t *testing.T) { + tests := []struct { + input string + output context + }{ + { + ``, + context{}, + }, + { + `Hello, World!`, + context{}, + }, + { + // An orphaned "<" is OK. + `I <3 Ponies!`, + context{}, + }, + { + `<a`, + context{state: stateTag}, + }, + { + `<a `, + context{state: stateTag}, + }, + { + `<a>`, + context{state: stateText}, + }, + { + `<a href`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a on`, + context{state: stateAttrName, attr: attrScript}, + }, + { + `<a href `, + context{state: stateAfterName, attr: attrURL}, + }, + { + `<a style = `, + context{state: stateBeforeValue, attr: attrStyle}, + }, + { + `<a href=`, + context{state: stateBeforeValue, attr: attrURL}, + }, + { + `<a href=x`, + context{state: stateURL, delim: delimSpaceOrTagEnd, urlPart: urlPartPreQuery}, + }, + { + `<a href=x `, + context{state: stateTag}, + }, + { + `<a href=>`, + context{state: stateText}, + }, + { + `<a href=x>`, + context{state: stateText}, + }, + { + `<a href ='`, + context{state: stateURL, delim: delimSingleQuote}, + }, + { + `<a href=''`, + context{state: stateTag}, + }, + { + `<a href= "`, + context{state: stateURL, delim: delimDoubleQuote}, + }, + { + `<a href=""`, + context{state: stateTag}, + }, + { + `<a title="`, + context{state: stateAttr, delim: delimDoubleQuote}, + }, + { + `<a HREF='http:`, + context{state: stateURL, delim: delimSingleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a Href='/`, + context{state: stateURL, delim: delimSingleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a href='"`, + context{state: stateURL, delim: delimSingleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a href="'`, + context{state: stateURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a href=''`, + context{state: stateURL, delim: delimSingleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a href=""`, + context{state: stateURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a href=""`, + context{state: stateURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a href="`, + context{state: stateURL, delim: delimSpaceOrTagEnd, urlPart: urlPartPreQuery}, + }, + { + `<img alt="1">`, + context{state: stateText}, + }, + { + `<img alt="1>"`, + context{state: stateTag}, + }, + { + `<img alt="1>">`, + context{state: stateText}, + }, + { + `<input checked type="checkbox"`, + context{state: stateTag}, + }, + { + `<a onclick="`, + context{state: stateJS, delim: delimDoubleQuote}, + }, + { + `<a onclick="//foo`, + context{state: stateJSLineCmt, delim: delimDoubleQuote}, + }, + { + "<a onclick='//\n", + context{state: stateJS, delim: delimSingleQuote}, + }, + { + "<a onclick='//\r\n", + context{state: stateJS, delim: delimSingleQuote}, + }, + { + "<a onclick='//\u2028", + context{state: stateJS, delim: delimSingleQuote}, + }, + { + `<a onclick="/*`, + context{state: stateJSBlockCmt, delim: delimDoubleQuote}, + }, + { + `<a onclick="/*/`, + context{state: stateJSBlockCmt, delim: delimDoubleQuote}, + }, + { + `<a onclick="/**/`, + context{state: stateJS, delim: delimDoubleQuote}, + }, + { + `<a onkeypress=""`, + context{state: stateJSDqStr, delim: delimDoubleQuote}, + }, + { + `<a onclick='"foo"`, + context{state: stateJS, delim: delimSingleQuote, jsCtx: jsCtxDivOp}, + }, + { + `<a onclick='foo'`, + context{state: stateJS, delim: delimSpaceOrTagEnd, jsCtx: jsCtxDivOp}, + }, + { + `<a onclick='foo`, + context{state: stateJSSqStr, delim: delimSpaceOrTagEnd}, + }, + { + `<a onclick=""foo'`, + context{state: stateJSDqStr, delim: delimDoubleQuote}, + }, + { + `<a onclick="'foo"`, + context{state: stateJSSqStr, delim: delimDoubleQuote}, + }, + { + `<A ONCLICK="'`, + context{state: stateJSSqStr, delim: delimDoubleQuote}, + }, + { + `<a onclick="/`, + context{state: stateJSRegexp, delim: delimDoubleQuote}, + }, + { + `<a onclick="'foo'`, + context{state: stateJS, delim: delimDoubleQuote, jsCtx: jsCtxDivOp}, + }, + { + `<a onclick="'foo\'`, + context{state: stateJSSqStr, delim: delimDoubleQuote}, + }, + { + `<a onclick="'foo\'`, + context{state: stateJSSqStr, delim: delimDoubleQuote}, + }, + { + `<a onclick="/foo/`, + context{state: stateJS, delim: delimDoubleQuote, jsCtx: jsCtxDivOp}, + }, + { + `<script>/foo/ /=`, + context{state: stateJS, element: elementScript}, + }, + { + `<a onclick="1 /foo`, + context{state: stateJS, delim: delimDoubleQuote, jsCtx: jsCtxDivOp}, + }, + { + `<a onclick="1 /*c*/ /foo`, + context{state: stateJS, delim: delimDoubleQuote, jsCtx: jsCtxDivOp}, + }, + { + `<a onclick="/foo[/]`, + context{state: stateJSRegexp, delim: delimDoubleQuote}, + }, + { + `<a onclick="/foo\/`, + context{state: stateJSRegexp, delim: delimDoubleQuote}, + }, + { + `<a onclick="/foo/`, + context{state: stateJS, delim: delimDoubleQuote, jsCtx: jsCtxDivOp}, + }, + { + `<input checked style="`, + context{state: stateCSS, delim: delimDoubleQuote}, + }, + { + `<a style="//`, + context{state: stateCSSLineCmt, delim: delimDoubleQuote}, + }, + { + `<a style="//</script>`, + context{state: stateCSSLineCmt, delim: delimDoubleQuote}, + }, + { + "<a style='//\n", + context{state: stateCSS, delim: delimSingleQuote}, + }, + { + "<a style='//\r", + context{state: stateCSS, delim: delimSingleQuote}, + }, + { + `<a style="/*`, + context{state: stateCSSBlockCmt, delim: delimDoubleQuote}, + }, + { + `<a style="/*/`, + context{state: stateCSSBlockCmt, delim: delimDoubleQuote}, + }, + { + `<a style="/**/`, + context{state: stateCSS, delim: delimDoubleQuote}, + }, + { + `<a style="background: '`, + context{state: stateCSSSqStr, delim: delimDoubleQuote}, + }, + { + `<a style="background: "`, + context{state: stateCSSDqStr, delim: delimDoubleQuote}, + }, + { + `<a style="background: '/foo?img=`, + context{state: stateCSSSqStr, delim: delimDoubleQuote, urlPart: urlPartQueryOrFrag}, + }, + { + `<a style="background: '/`, + context{state: stateCSSSqStr, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a style="background: url("/`, + context{state: stateCSSDqURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a style="background: url('/`, + context{state: stateCSSSqURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a style="background: url('/)`, + context{state: stateCSSSqURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a style="background: url('/ `, + context{state: stateCSSSqURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a style="background: url(/`, + context{state: stateCSSURL, delim: delimDoubleQuote, urlPart: urlPartPreQuery}, + }, + { + `<a style="background: url( `, + context{state: stateCSSURL, delim: delimDoubleQuote}, + }, + { + `<a style="background: url( /image?name=`, + context{state: stateCSSURL, delim: delimDoubleQuote, urlPart: urlPartQueryOrFrag}, + }, + { + `<a style="background: url(x)`, + context{state: stateCSS, delim: delimDoubleQuote}, + }, + { + `<a style="background: url('x'`, + context{state: stateCSS, delim: delimDoubleQuote}, + }, + { + `<a style="background: url( x `, + context{state: stateCSS, delim: delimDoubleQuote}, + }, + { + `<!-- foo`, + context{state: stateHTMLCmt}, + }, + { + `<!-->`, + context{state: stateHTMLCmt}, + }, + { + `<!--->`, + context{state: stateHTMLCmt}, + }, + { + `<!-- foo -->`, + context{state: stateText}, + }, + { + `<script`, + context{state: stateTag, element: elementScript}, + }, + { + `<script `, + context{state: stateTag, element: elementScript}, + }, + { + `<script src="foo.js" `, + context{state: stateTag, element: elementScript}, + }, + { + `<script src='foo.js' `, + context{state: stateTag, element: elementScript}, + }, + { + `<script type=text/javascript `, + context{state: stateTag, element: elementScript}, + }, + { + `<script>foo`, + context{state: stateJS, jsCtx: jsCtxDivOp, element: elementScript}, + }, + { + `<script>foo</script>`, + context{state: stateText}, + }, + { + `<script>foo</script><!--`, + context{state: stateHTMLCmt}, + }, + { + `<script>document.write("<p>foo</p>");`, + context{state: stateJS, element: elementScript}, + }, + { + `<script>document.write("<p>foo<\/script>");`, + context{state: stateJS, element: elementScript}, + }, + { + `<script>document.write("<script>alert(1)</script>");`, + context{state: stateText}, + }, + { + `<Script>`, + context{state: stateJS, element: elementScript}, + }, + { + `<SCRIPT>foo`, + context{state: stateJS, jsCtx: jsCtxDivOp, element: elementScript}, + }, + { + `<textarea>value`, + context{state: stateRCDATA, element: elementTextarea}, + }, + { + `<textarea>value</TEXTAREA>`, + context{state: stateText}, + }, + { + `<textarea name=html><b`, + context{state: stateRCDATA, element: elementTextarea}, + }, + { + `<title>value`, + context{state: stateRCDATA, element: elementTitle}, + }, + { + `<style>value`, + context{state: stateCSS, element: elementStyle}, + }, + { + `<a xlink:href`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a xmlns`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a xmlns:foo`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a xmlnsxyz`, + context{state: stateAttrName}, + }, + { + `<a data-url`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a data-iconUri`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a data-urlItem`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a g:`, + context{state: stateAttrName}, + }, + { + `<a g:url`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a g:iconUri`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a g:urlItem`, + context{state: stateAttrName, attr: attrURL}, + }, + { + `<a g:value`, + context{state: stateAttrName}, + }, + { + `<a svg:style='`, + context{state: stateCSS, delim: delimSingleQuote}, + }, + { + `<svg:font-face`, + context{state: stateTag}, + }, + { + `<svg:a svg:onclick="`, + context{state: stateJS, delim: delimDoubleQuote}, + }, + } - buffer := new(bytes.Buffer) + for _, test := range tests { + b, e := []byte(test.input), newEscaper(nil) + c := e.escapeText(context{}, &parse.TextNode{parse.NodeText, b}) + if !test.output.eq(c) { + t.Errorf("input %q: want context\n\t%v\ngot\n\t%v", test.input, test.output, c) + continue + } + if test.input != string(b) { + t.Errorf("input %q: text node was modified: want %q got %q", test.input, test.input, b) + continue + } + } +} - err = tmpl.Execute(buffer, testData) - if err != nil { - t.Errorf("%s: template execution failed: %s", name, err) +func TestEnsurePipelineContains(t *testing.T) { + tests := []struct { + input, output string + ids []string + }{ + { + "{{.X}}", + "[(command: [F=[X]])]", + []string{}, + }, + { + "{{.X | html}}", + "[(command: [F=[X]]) (command: [I=html])]", + []string{}, + }, + { + "{{.X}}", + "[(command: [F=[X]]) (command: [I=html])]", + []string{"html"}, + }, + { + "{{.X | html}}", + "[(command: [F=[X]]) (command: [I=html]) (command: [I=urlquery])]", + []string{"urlquery"}, + }, + { + "{{.X | html | urlquery}}", + "[(command: [F=[X]]) (command: [I=html]) (command: [I=urlquery])]", + []string{"urlquery"}, + }, + { + "{{.X | html | urlquery}}", + "[(command: [F=[X]]) (command: [I=html]) (command: [I=urlquery])]", + []string{"html", "urlquery"}, + }, + { + "{{.X | html | urlquery}}", + "[(command: [F=[X]]) (command: [I=html]) (command: [I=urlquery])]", + []string{"html"}, + }, + { + "{{.X | urlquery}}", + "[(command: [F=[X]]) (command: [I=html]) (command: [I=urlquery])]", + []string{"html", "urlquery"}, + }, + { + "{{.X | html | print}}", + "[(command: [F=[X]]) (command: [I=urlquery]) (command: [I=html]) (command: [I=print])]", + []string{"urlquery", "html"}, + }, + } + for _, test := range tests { + tmpl := template.Must(template.New("test").Parse(test.input)) + action, ok := (tmpl.Tree.Root.Nodes[0].(*parse.ActionNode)) + if !ok { + t.Errorf("First node is not an action: %s", test.input) continue } + pipe := action.Pipe + ensurePipelineContains(pipe, test.ids) + got := pipe.String() + if got != test.output { + t.Errorf("%s, %v: want\n\t%s\ngot\n\t%s", test.input, test.ids, test.output, got) + } + } +} - output := testCase.output - actual := buffer.String() - if output != actual { - t.Errorf("%s: escaped output: %q != %q", - name, output, actual) +func expectExecuteFailure(t *testing.T, b *bytes.Buffer) { + if x := recover(); x != nil { + if b.Len() != 0 { + t.Errorf("output on buffer: %q", b.String()) } + } else { + t.Errorf("unescaped template executed") + } +} + +func TestEscapeErrorsNotIgnorable(t *testing.T) { + var b bytes.Buffer + tmpl := template.Must(template.New("dangerous").Parse("<a")) + Escape(tmpl) + defer expectExecuteFailure(t, &b) + tmpl.Execute(&b, nil) +} + +func TestEscapeSetErrorsNotIgnorable(t *testing.T) { + s, err := (&template.Set{}).Parse(`{{define "t"}}<a{{end}}`) + if err != nil { + t.Errorf("failed to parse set: %q", err) + } + EscapeSet(s, "t") + var b bytes.Buffer + defer expectExecuteFailure(t, &b) + s.Execute(&b, "t", nil) +} + +func TestRedundantFuncs(t *testing.T) { + inputs := []interface{}{ + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./` + + `0123456789:;<=>?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\]^_` + + "`abcdefghijklmno" + + "pqrstuvwxyz{|}~\x7f" + + "\u00A0\u0100\u2028\u2029\ufeff\ufdec\ufffd\uffff\U0001D11E" + + "&%22\\", + CSS(`a[href =~ "//example.com"]#foo`), + HTML(`Hello, <b>World</b> &tc!`), + HTMLAttr(` dir="ltr"`), + JS(`c && alert("Hello, World!");`), + JSStr(`Hello, World & O'Reilly\x21`), + URL(`greeting=H%69&addressee=(World)`), + } + + for n0, m := range redundantFuncs { + f0 := funcMap[n0].(func(...interface{}) string) + for n1, _ := range m { + f1 := funcMap[n1].(func(...interface{}) string) + for _, input := range inputs { + want := f0(input) + if got := f1(want); want != got { + t.Errorf("%s %s with %T %q: want\n\t%q,\ngot\n\t%q", n0, n1, input, input, want, got) + } + } + } + } +} + +func BenchmarkEscapedExecute(b *testing.B) { + tmpl := template.Must(Escape(template.Must(template.New("t").Parse(`<a onclick="alert('{{.}}')">{{.}}</a>`)))) + var buf bytes.Buffer + b.ResetTimer() + for i := 0; i < b.N; i++ { + tmpl.Execute(&buf, "foo & 'bar' & baz") + buf.Reset() } } diff --git a/libgo/go/exp/template/html/html.go b/libgo/go/exp/template/html/html.go new file mode 100644 index 0000000..91bb1b1 --- /dev/null +++ b/libgo/go/exp/template/html/html.go @@ -0,0 +1,257 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "fmt" + "strings" + "utf8" +) + +// htmlNospaceEscaper escapes for inclusion in unquoted attribute values. +func htmlNospaceEscaper(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeHTML { + return htmlReplacer(stripTags(s), htmlNospaceNormReplacementTable, false) + } + return htmlReplacer(s, htmlNospaceReplacementTable, false) +} + +// attrEscaper escapes for inclusion in quoted attribute values. +func attrEscaper(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeHTML { + return htmlReplacer(stripTags(s), htmlNormReplacementTable, true) + } + return htmlReplacer(s, htmlReplacementTable, true) +} + +// rcdataEscaper escapes for inclusion in an RCDATA element body. +func rcdataEscaper(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeHTML { + return htmlReplacer(s, htmlNormReplacementTable, true) + } + return htmlReplacer(s, htmlReplacementTable, true) +} + +// htmlEscaper escapes for inclusion in HTML text. +func htmlEscaper(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeHTML { + return s + } + return htmlReplacer(s, htmlReplacementTable, true) +} + +// htmlReplacementTable contains the runes that need to be escaped +// inside a quoted attribute value or in a text node. +var htmlReplacementTable = []string{ + // http://www.w3.org/TR/html5/tokenization.html#attribute-value-unquoted-state: " + // U+0000 NULL Parse error. Append a U+FFFD REPLACEMENT + // CHARACTER character to the current attribute's value. + // " + // and similarly + // http://www.w3.org/TR/html5/tokenization.html#before-attribute-value-state + 0: "\uFFFD", + '"': """, + '&': "&", + '\'': "'", + '+': "+", + '<': "<", + '>': ">", +} + +// htmlNormReplacementTable is like htmlReplacementTable but without '&' to +// avoid over-encoding existing entities. +var htmlNormReplacementTable = []string{ + 0: "\uFFFD", + '"': """, + '\'': "'", + '+': "+", + '<': "<", + '>': ">", +} + +// htmlNospaceReplacementTable contains the runes that need to be escaped +// inside an unquoted attribute value. +// The set of runes escaped is the union of the HTML specials and +// those determined by running the JS below in browsers: +// <div id=d></div> +// <script>(function () { +// var a = [], d = document.getElementById("d"), i, c, s; +// for (i = 0; i < 0x10000; ++i) { +// c = String.fromCharCode(i); +// d.innerHTML = "<span title=" + c + "lt" + c + "></span>" +// s = d.getElementsByTagName("SPAN")[0]; +// if (!s || s.title !== c + "lt" + c) { a.push(i.toString(16)); } +// } +// document.write(a.join(", ")); +// })()</script> +var htmlNospaceReplacementTable = []string{ + 0: "�", + '\t': "	", + '\n': " ", + '\v': "", + '\f': "", + '\r': " ", + ' ': " ", + '"': """, + '&': "&", + '\'': "'", + '+': "+", + '<': "<", + '=': "=", + '>': ">", + // A parse error in the attribute value (unquoted) and + // before attribute value states. + // Treated as a quoting character by IE. + '`': "`", +} + +// htmlNospaceNormReplacementTable is like htmlNospaceReplacementTable but +// without '&' to avoid over-encoding existing entities. +var htmlNospaceNormReplacementTable = []string{ + 0: "�", + '\t': "	", + '\n': " ", + '\v': "", + '\f': "", + '\r': " ", + ' ': " ", + '"': """, + '\'': "'", + '+': "+", + '<': "<", + '=': "=", + '>': ">", + // A parse error in the attribute value (unquoted) and + // before attribute value states. + // Treated as a quoting character by IE. + '`': "`", +} + +// htmlReplacer returns s with runes replaced acccording to replacementTable +// and when badRunes is true, certain bad runes are allowed through unescaped. +func htmlReplacer(s string, replacementTable []string, badRunes bool) string { + written, b := 0, new(bytes.Buffer) + for i, r := range s { + if r < len(replacementTable) { + if repl := replacementTable[r]; len(repl) != 0 { + b.WriteString(s[written:i]) + b.WriteString(repl) + // Valid as long as replacementTable doesn't + // include anything above 0x7f. + written = i + utf8.RuneLen(r) + } + } else if badRunes { + // No-op. + // IE does not allow these ranges in unquoted attrs. + } else if 0xfdd0 <= r && r <= 0xfdef || 0xfff0 <= r && r <= 0xffff { + fmt.Fprintf(b, "%s&#x%x;", s[written:i], r) + written = i + utf8.RuneLen(r) + } + } + if written == 0 { + return s + } + b.WriteString(s[written:]) + return b.String() +} + +// stripTags takes a snippet of HTML and returns only the text content. +// For example, `<b>¡Hi!</b> <script>...</script>` -> `¡Hi! `. +func stripTags(html string) string { + var b bytes.Buffer + s, c, i, allText := []byte(html), context{}, 0, true + // Using the transition funcs helps us avoid mangling + // `<div title="1>2">` or `I <3 Ponies!`. + for i != len(s) { + if c.delim == delimNone { + st := c.state + // Use RCDATA instead of parsing into JS or CSS styles. + if c.element != elementNone && !isInTag(st) { + st = stateRCDATA + } + d, nread := transitionFunc[st](c, s[i:]) + i1 := i + nread + if c.state == stateText || c.state == stateRCDATA { + // Emit text up to the start of the tag or comment. + j := i1 + if d.state != c.state { + for j1 := j - 1; j1 >= i; j1-- { + if s[j1] == '<' { + j = j1 + break + } + } + } + b.Write(s[i:j]) + } else { + allText = false + } + c, i = d, i1 + continue + } + i1 := i + bytes.IndexAny(s[i:], delimEnds[c.delim]) + if i1 < i { + break + } + if c.delim != delimSpaceOrTagEnd { + // Consume any quote. + i1++ + } + c, i = context{state: stateTag, element: c.element}, i1 + } + if allText { + return html + } else if c.state == stateText || c.state == stateRCDATA { + b.Write(s[i:]) + } + return b.String() +} + +// htmlNameFilter accepts valid parts of an HTML attribute or tag name or +// a known-safe HTML attribute. +func htmlNameFilter(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeHTMLAttr { + return s + } + if len(s) == 0 { + // Avoid violation of structure preservation. + // <input checked {{.K}}={{.V}}>. + // Without this, if .K is empty then .V is the value of + // checked, but otherwise .V is the value of the attribute + // named .K. + return filterFailsafe + } + s = strings.ToLower(s) + if t := attrType(s); t != contentTypePlain { + // TODO: Split attr and element name part filters so we can whitelist + // attributes. + return filterFailsafe + } + for _, r := range s { + switch { + case '0' <= r && r <= '9': + case 'a' <= r && r <= 'z': + default: + return filterFailsafe + } + } + return s +} + +// commentEscaper returns the empty string regardless of input. +// Comment content does not correspond to any parsed structure or +// human-readable content, so the simplest and most secure policy is to drop +// content interpolated into comments. +// This approach is equally valid whether or not static comment content is +// removed from the template. +func commentEscaper(args ...interface{}) string { + return "" +} diff --git a/libgo/go/exp/template/html/html_test.go b/libgo/go/exp/template/html/html_test.go new file mode 100644 index 0000000..e178d0f --- /dev/null +++ b/libgo/go/exp/template/html/html_test.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "html" + "strings" + "testing" +) + +func TestHTMLNospaceEscaper(t *testing.T) { + input := ("\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./` + + `0123456789:;<=>?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\]^_` + + "`abcdefghijklmno" + + "pqrstuvwxyz{|}~\x7f" + + "\u00A0\u0100\u2028\u2029\ufeff\ufdec\U0001D11E") + + want := ("�\x01\x02\x03\x04\x05\x06\x07" + + "\x08	  \x0E\x0F" + + "\x10\x11\x12\x13\x14\x15\x16\x17" + + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./` + + `0123456789:;<=>?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\]^_` + + ``abcdefghijklmno` + + `pqrstuvwxyz{|}~` + "\u007f" + + "\u00A0\u0100\u2028\u2029\ufeff\U0001D11E") + + got := htmlNospaceEscaper(input) + if got != want { + t.Errorf("encode: want\n\t%q\nbut got\n\t%q", want, got) + } + + got, want = html.UnescapeString(got), strings.Replace(input, "\x00", "\ufffd", 1) + if want != got { + t.Errorf("decode: want\n\t%q\nbut got\n\t%q", want, got) + } +} + +func TestStripTags(t *testing.T) { + tests := []struct { + input, want string + }{ + {"", ""}, + {"Hello, World!", "Hello, World!"}, + {"foo&bar", "foo&bar"}, + {`Hello <a href="www.example.com/">World</a>!`, "Hello World!"}, + {"Foo <textarea>Bar</textarea> Baz", "Foo Bar Baz"}, + {"Foo <!-- Bar --> Baz", "Foo Baz"}, + {"<", "<"}, + {"foo < bar", "foo < bar"}, + {`Foo<script type="text/javascript">alert(1337)</script>Bar`, "FooBar"}, + {`Foo<div title="1>2">Bar`, "FooBar"}, + {`I <3 Ponies!`, `I <3 Ponies!`}, + {`<script>foo()</script>`, ``}, + } + + for _, test := range tests { + if got := stripTags(test.input); got != test.want { + t.Errorf("%q: want %q, got %q", test.input, test.want, got) + } + } +} + +func BenchmarkHTMLNospaceEscaper(b *testing.B) { + for i := 0; i < b.N; i++ { + htmlNospaceEscaper("The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>") + } +} + +func BenchmarkHTMLNospaceEscaperNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + htmlNospaceEscaper("The_quick,_brown_fox_jumps_over_the_lazy_dog.") + } +} + +func BenchmarkStripTags(b *testing.B) { + for i := 0; i < b.N; i++ { + stripTags("The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>") + } +} + +func BenchmarkStripTagsNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + stripTags("The quick, brown fox jumps over the lazy dog.") + } +} diff --git a/libgo/go/exp/template/html/js.go b/libgo/go/exp/template/html/js.go new file mode 100644 index 0000000..98c2ac5 --- /dev/null +++ b/libgo/go/exp/template/html/js.go @@ -0,0 +1,346 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "fmt" + "json" + "strings" + "utf8" +) + +// nextJSCtx returns the context that determines whether a slash after the +// given run of tokens tokens starts a regular expression instead of a division +// operator: / or /=. +// +// This assumes that the token run does not include any string tokens, comment +// tokens, regular expression literal tokens, or division operators. +// +// This fails on some valid but nonsensical JavaScript programs like +// "x = ++/foo/i" which is quite different than "x++/foo/i", but is not known to +// fail on any known useful programs. It is based on the draft +// JavaScript 2.0 lexical grammar and requires one token of lookbehind: +// http://www.mozilla.org/js/language/js20-2000-07/rationale/syntax.html +func nextJSCtx(s []byte, preceding jsCtx) jsCtx { + s = bytes.TrimRight(s, "\t\n\f\r \u2028\u2029") + if len(s) == 0 { + return preceding + } + + // All cases below are in the single-byte UTF-8 group. + switch c, n := s[len(s)-1], len(s); c { + case '+', '-': + // ++ and -- are not regexp preceders, but + and - are whether + // they are used as infix or prefix operators. + start := n - 1 + // Count the number of adjacent dashes or pluses. + for start > 0 && s[start-1] == c { + start-- + } + if (n-start)&1 == 1 { + // Reached for trailing minus signs since "---" is the + // same as "-- -". + return jsCtxRegexp + } + return jsCtxDivOp + case '.': + // Handle "42." + if n != 1 && '0' <= s[n-2] && s[n-2] <= '9' { + return jsCtxDivOp + } + return jsCtxRegexp + // Suffixes for all punctuators from section 7.7 of the language spec + // that only end binary operators not handled above. + case ',', '<', '>', '=', '*', '%', '&', '|', '^', '?': + return jsCtxRegexp + // Suffixes for all punctuators from section 7.7 of the language spec + // that are prefix operators not handled above. + case '!', '~': + return jsCtxRegexp + // Matches all the punctuators from section 7.7 of the language spec + // that are open brackets not handled above. + case '(', '[': + return jsCtxRegexp + // Matches all the punctuators from section 7.7 of the language spec + // that precede expression starts. + case ':', ';', '{': + return jsCtxRegexp + // CAVEAT: the close punctuators ('}', ']', ')') precede div ops and + // are handled in the default except for '}' which can precede a + // division op as in + // ({ valueOf: function () { return 42 } } / 2 + // which is valid, but, in practice, developers don't divide object + // literals, so our heuristic works well for code like + // function () { ... } /foo/.test(x) && sideEffect(); + // The ')' punctuator can precede a regular expression as in + // if (b) /foo/.test(x) && ... + // but this is much less likely than + // (a + b) / c + case '}': + return jsCtxRegexp + default: + // Look for an IdentifierName and see if it is a keyword that + // can precede a regular expression. + j := n + for j > 0 && isJSIdentPart(int(s[j-1])) { + j-- + } + if regexpPrecederKeywords[string(s[j:])] { + return jsCtxRegexp + } + } + // Otherwise is a punctuator not listed above, or + // a string which precedes a div op, or an identifier + // which precedes a div op. + return jsCtxDivOp +} + +// regexPrecederKeywords is a set of reserved JS keywords that can precede a +// regular expression in JS source. +var regexpPrecederKeywords = map[string]bool{ + "break": true, + "case": true, + "continue": true, + "delete": true, + "do": true, + "else": true, + "finally": true, + "in": true, + "instanceof": true, + "return": true, + "throw": true, + "try": true, + "typeof": true, + "void": true, +} + +// jsValEscaper escapes its inputs to a JS Expression (section 11.14) that has +// nether side-effects nor free variables outside (NaN, Infinity). +func jsValEscaper(args ...interface{}) string { + var a interface{} + if len(args) == 1 { + a = args[0] + switch t := a.(type) { + case JS: + return string(t) + case JSStr: + // TODO: normalize quotes. + return `"` + string(t) + `"` + case json.Marshaler: + // Do not treat as a Stringer. + case fmt.Stringer: + a = t.String() + } + } else { + a = fmt.Sprint(args...) + } + // TODO: detect cycles before calling Marshal which loops infinitely on + // cyclic data. This may be an unnacceptable DoS risk. + + b, err := json.Marshal(a) + if err != nil { + // Put a space before comment so that if it is flush against + // a division operator it is not turned into a line comment: + // x/{{y}} + // turning into + // x//* error marshalling y: + // second line of error message */null + return fmt.Sprintf(" /* %s */null ", strings.Replace(err.String(), "*/", "* /", -1)) + } + + // TODO: maybe post-process output to prevent it from containing + // "<!--", "-->", "<![CDATA[", "]]>", or "</script" + // in case custom marshallers produce output containing those. + + // TODO: Maybe abbreviate \u00ab to \xab to produce more compact output. + if len(b) == 0 { + // In, `x=y/{{.}}*z` a json.Marshaler that produces "" should + // not cause the output `x=y/*z`. + return " null " + } + first, _ := utf8.DecodeRune(b) + last, _ := utf8.DecodeLastRune(b) + var buf bytes.Buffer + // Prevent IdentifierNames and NumericLiterals from running into + // keywords: in, instanceof, typeof, void + pad := isJSIdentPart(first) || isJSIdentPart(last) + if pad { + buf.WriteByte(' ') + } + written := 0 + // Make sure that json.Marshal escapes codepoints U+2028 & U+2029 + // so it falls within the subset of JSON which is valid JS. + for i := 0; i < len(b); { + rune, n := utf8.DecodeRune(b[i:]) + repl := "" + if rune == 0x2028 { + repl = `\u2028` + } else if rune == 0x2029 { + repl = `\u2029` + } + if repl != "" { + buf.Write(b[written:i]) + buf.WriteString(repl) + written = i + n + } + i += n + } + if buf.Len() != 0 { + buf.Write(b[written:]) + if pad { + buf.WriteByte(' ') + } + b = buf.Bytes() + } + return string(b) +} + +// jsStrEscaper produces a string that can be included between quotes in +// JavaScript source, in JavaScript embedded in an HTML5 <script> element, +// or in an HTML5 event handler attribute such as onclick. +func jsStrEscaper(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeJSStr { + return replace(s, jsStrNormReplacementTable) + } + return replace(s, jsStrReplacementTable) +} + +// jsRegexpEscaper behaves like jsStrEscaper but escapes regular expression +// specials so the result is treated literally when included in a regular +// expression literal. /foo{{.X}}bar/ matches the string "foo" followed by +// the literal text of {{.X}} followed by the string "bar". +func jsRegexpEscaper(args ...interface{}) string { + s, _ := stringify(args...) + s = replace(s, jsRegexpReplacementTable) + if s == "" { + // /{{.X}}/ should not produce a line comment when .X == "". + return "(?:)" + } + return s +} + +// replace replaces each rune r of s with replacementTable[r], provided that +// r < len(replacementTable). If replacementTable[r] is the empty string then +// no replacement is made. +// It also replaces runes U+2028 and U+2029 with the raw strings `\u2028` and +// `\u2029`. +func replace(s string, replacementTable []string) string { + var b bytes.Buffer + written := 0 + for i, r := range s { + var repl string + switch { + case r < len(replacementTable) && replacementTable[r] != "": + repl = replacementTable[r] + case r == '\u2028': + repl = `\u2028` + case r == '\u2029': + repl = `\u2029` + default: + continue + } + b.WriteString(s[written:i]) + b.WriteString(repl) + written = i + utf8.RuneLen(r) + } + if written == 0 { + return s + } + b.WriteString(s[written:]) + return b.String() +} + +var jsStrReplacementTable = []string{ + 0: `\0`, + '\t': `\t`, + '\n': `\n`, + '\v': `\x0b`, // "\v" == "v" on IE 6. + '\f': `\f`, + '\r': `\r`, + // Encode HTML specials as hex so the output can be embedded + // in HTML attributes without further encoding. + '"': `\x22`, + '&': `\x26`, + '\'': `\x27`, + '+': `\x2b`, + '/': `\/`, + '<': `\x3c`, + '>': `\x3e`, + '\\': `\\`, +} + +// jsStrNormReplacementTable is like jsStrReplacementTable but does not +// overencode existing escapes since this table has no entry for `\`. +var jsStrNormReplacementTable = []string{ + 0: `\0`, + '\t': `\t`, + '\n': `\n`, + '\v': `\x0b`, // "\v" == "v" on IE 6. + '\f': `\f`, + '\r': `\r`, + // Encode HTML specials as hex so the output can be embedded + // in HTML attributes without further encoding. + '"': `\x22`, + '&': `\x26`, + '\'': `\x27`, + '+': `\x2b`, + '/': `\/`, + '<': `\x3c`, + '>': `\x3e`, +} + +var jsRegexpReplacementTable = []string{ + 0: `\0`, + '\t': `\t`, + '\n': `\n`, + '\v': `\x0b`, // "\v" == "v" on IE 6. + '\f': `\f`, + '\r': `\r`, + // Encode HTML specials as hex so the output can be embedded + // in HTML attributes without further encoding. + '"': `\x22`, + '$': `\$`, + '&': `\x26`, + '\'': `\x27`, + '(': `\(`, + ')': `\)`, + '*': `\*`, + '+': `\x2b`, + '-': `\-`, + '.': `\.`, + '/': `\/`, + '<': `\x3c`, + '>': `\x3e`, + '?': `\?`, + '[': `\[`, + '\\': `\\`, + ']': `\]`, + '^': `\^`, + '{': `\{`, + '|': `\|`, + '}': `\}`, +} + +// isJSIdentPart returns whether the given rune is a JS identifier part. +// It does not handle all the non-Latin letters, joiners, and combining marks, +// but it does handle every codepoint that can occur in a numeric literal or +// a keyword. +func isJSIdentPart(rune int) bool { + switch { + case '$' == rune: + return true + case '0' <= rune && rune <= '9': + return true + case 'A' <= rune && rune <= 'Z': + return true + case '_' == rune: + return true + case 'a' <= rune && rune <= 'z': + return true + } + return false +} diff --git a/libgo/go/exp/template/html/js_test.go b/libgo/go/exp/template/html/js_test.go new file mode 100644 index 0000000..e7764054 --- /dev/null +++ b/libgo/go/exp/template/html/js_test.go @@ -0,0 +1,401 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "math" + "strings" + "testing" +) + +func TestNextJsCtx(t *testing.T) { + tests := []struct { + jsCtx jsCtx + s string + }{ + // Statement terminators precede regexps. + {jsCtxRegexp, ";"}, + // This is not airtight. + // ({ valueOf: function () { return 1 } } / 2) + // is valid JavaScript but in practice, devs do not do this. + // A block followed by a statement starting with a RegExp is + // much more common: + // while (x) {...} /foo/.test(x) || panic() + {jsCtxRegexp, "}"}, + // But member, call, grouping, and array expression terminators + // precede div ops. + {jsCtxDivOp, ")"}, + {jsCtxDivOp, "]"}, + // At the start of a primary expression, array, or expression + // statement, expect a regexp. + {jsCtxRegexp, "("}, + {jsCtxRegexp, "["}, + {jsCtxRegexp, "{"}, + // Assignment operators precede regexps as do all exclusively + // prefix and binary operators. + {jsCtxRegexp, "="}, + {jsCtxRegexp, "+="}, + {jsCtxRegexp, "*="}, + {jsCtxRegexp, "*"}, + {jsCtxRegexp, "!"}, + // Whether the + or - is infix or prefix, it cannot precede a + // div op. + {jsCtxRegexp, "+"}, + {jsCtxRegexp, "-"}, + // An incr/decr op precedes a div operator. + // This is not airtight. In (g = ++/h/i) a regexp follows a + // pre-increment operator, but in practice devs do not try to + // increment or decrement regular expressions. + // (g++/h/i) where ++ is a postfix operator on g is much more + // common. + {jsCtxDivOp, "--"}, + {jsCtxDivOp, "++"}, + {jsCtxDivOp, "x--"}, + // When we have many dashes or pluses, then they are grouped + // left to right. + {jsCtxRegexp, "x---"}, // A postfix -- then a -. + // return followed by a slash returns the regexp literal or the + // slash starts a regexp literal in an expression statement that + // is dead code. + {jsCtxRegexp, "return"}, + {jsCtxRegexp, "return "}, + {jsCtxRegexp, "return\t"}, + {jsCtxRegexp, "return\n"}, + {jsCtxRegexp, "return\u2028"}, + // Identifiers can be divided and cannot validly be preceded by + // a regular expressions. Semicolon insertion cannot happen + // between an identifier and a regular expression on a new line + // because the one token lookahead for semicolon insertion has + // to conclude that it could be a div binary op and treat it as + // such. + {jsCtxDivOp, "x"}, + {jsCtxDivOp, "x "}, + {jsCtxDivOp, "x\t"}, + {jsCtxDivOp, "x\n"}, + {jsCtxDivOp, "x\u2028"}, + {jsCtxDivOp, "preturn"}, + // Numbers precede div ops. + {jsCtxDivOp, "0"}, + // Dots that are part of a number are div preceders. + {jsCtxDivOp, "0."}, + } + + for _, test := range tests { + if nextJSCtx([]byte(test.s), jsCtxRegexp) != test.jsCtx { + t.Errorf("want %s got %q", test.jsCtx, test.s) + } + if nextJSCtx([]byte(test.s), jsCtxDivOp) != test.jsCtx { + t.Errorf("want %s got %q", test.jsCtx, test.s) + } + } + + if nextJSCtx([]byte(" "), jsCtxRegexp) != jsCtxRegexp { + t.Error("Blank tokens") + } + + if nextJSCtx([]byte(" "), jsCtxDivOp) != jsCtxDivOp { + t.Error("Blank tokens") + } +} + +func TestJSValEscaper(t *testing.T) { + tests := []struct { + x interface{} + js string + }{ + {int(42), " 42 "}, + {uint(42), " 42 "}, + {int16(42), " 42 "}, + {uint16(42), " 42 "}, + {int32(-42), " -42 "}, + {uint32(42), " 42 "}, + {int16(-42), " -42 "}, + {uint16(42), " 42 "}, + {int64(-42), " -42 "}, + {uint64(42), " 42 "}, + {uint64(1) << 53, " 9007199254740992 "}, + // ulp(1 << 53) > 1 so this loses precision in JS + // but it is still a representable integer literal. + {uint64(1)<<53 + 1, " 9007199254740993 "}, + {float32(1.0), " 1 "}, + {float32(-1.0), " -1 "}, + {float32(0.5), " 0.5 "}, + {float32(-0.5), " -0.5 "}, + {float32(1.0) / float32(256), " 0.00390625 "}, + {float32(0), " 0 "}, + {math.Copysign(0, -1), " -0 "}, + {float64(1.0), " 1 "}, + {float64(-1.0), " -1 "}, + {float64(0.5), " 0.5 "}, + {float64(-0.5), " -0.5 "}, + {float64(0), " 0 "}, + {math.Copysign(0, -1), " -0 "}, + {"", `""`}, + {"foo", `"foo"`}, + // Newlines. + {"\r\n\u2028\u2029", `"\r\n\u2028\u2029"`}, + // "\v" == "v" on IE 6 so use "\x0b" instead. + {"\t\x0b", `"\u0009\u000b"`}, + {struct{ X, Y int }{1, 2}, `{"X":1,"Y":2}`}, + {[]interface{}{}, "[]"}, + {[]interface{}{42, "foo", nil}, `[42,"foo",null]`}, + {[]string{"<!--", "</script>", "-->"}, `["\u003c!--","\u003c/script\u003e","--\u003e"]`}, + {"<!--", `"\u003c!--"`}, + {"-->", `"--\u003e"`}, + {"<![CDATA[", `"\u003c![CDATA["`}, + {"]]>", `"]]\u003e"`}, + {"</script", `"\u003c/script"`}, + {"\U0001D11E", "\"\U0001D11E\""}, // or "\uD834\uDD1E" + } + + for _, test := range tests { + if js := jsValEscaper(test.x); js != test.js { + t.Errorf("%+v: want\n\t%q\ngot\n\t%q", test.x, test.js, js) + } + // Make sure that escaping corner cases are not broken + // by nesting. + a := []interface{}{test.x} + want := "[" + strings.TrimSpace(test.js) + "]" + if js := jsValEscaper(a); js != want { + t.Errorf("%+v: want\n\t%q\ngot\n\t%q", a, want, js) + } + } +} + +func TestJSStrEscaper(t *testing.T) { + tests := []struct { + x interface{} + esc string + }{ + {"", ``}, + {"foo", `foo`}, + {"\u0000", `\0`}, + {"\t", `\t`}, + {"\n", `\n`}, + {"\r", `\r`}, + {"\u2028", `\u2028`}, + {"\u2029", `\u2029`}, + {"\\", `\\`}, + {"\\n", `\\n`}, + {"foo\r\nbar", `foo\r\nbar`}, + // Preserve attribute boundaries. + {`"`, `\x22`}, + {`'`, `\x27`}, + // Allow embedding in HTML without further escaping. + {`&`, `\x26amp;`}, + // Prevent breaking out of text node and element boundaries. + {"</script>", `\x3c\/script\x3e`}, + {"<![CDATA[", `\x3c![CDATA[`}, + {"]]>", `]]\x3e`}, + // http://dev.w3.org/html5/markup/aria/syntax.html#escaping-text-span + // "The text in style, script, title, and textarea elements + // must not have an escaping text span start that is not + // followed by an escaping text span end." + // Furthermore, spoofing an escaping text span end could lead + // to different interpretation of a </script> sequence otherwise + // masked by the escaping text span, and spoofing a start could + // allow regular text content to be interpreted as script + // allowing script execution via a combination of a JS string + // injection followed by an HTML text injection. + {"<!--", `\x3c!--`}, + {"-->", `--\x3e`}, + // From http://code.google.com/p/doctype/wiki/ArticleUtf7 + {"+ADw-script+AD4-alert(1)+ADw-/script+AD4-", + `\x2bADw-script\x2bAD4-alert(1)\x2bADw-\/script\x2bAD4-`, + }, + // Invalid UTF-8 sequence + {"foo\xA0bar", "foo\xA0bar"}, + // Invalid unicode scalar value. + {"foo\xed\xa0\x80bar", "foo\xed\xa0\x80bar"}, + } + + for _, test := range tests { + esc := jsStrEscaper(test.x) + if esc != test.esc { + t.Errorf("%q: want %q got %q", test.x, test.esc, esc) + } + } +} + +func TestJSRegexpEscaper(t *testing.T) { + tests := []struct { + x interface{} + esc string + }{ + {"", `(?:)`}, + {"foo", `foo`}, + {"\u0000", `\0`}, + {"\t", `\t`}, + {"\n", `\n`}, + {"\r", `\r`}, + {"\u2028", `\u2028`}, + {"\u2029", `\u2029`}, + {"\\", `\\`}, + {"\\n", `\\n`}, + {"foo\r\nbar", `foo\r\nbar`}, + // Preserve attribute boundaries. + {`"`, `\x22`}, + {`'`, `\x27`}, + // Allow embedding in HTML without further escaping. + {`&`, `\x26amp;`}, + // Prevent breaking out of text node and element boundaries. + {"</script>", `\x3c\/script\x3e`}, + {"<![CDATA[", `\x3c!\[CDATA\[`}, + {"]]>", `\]\]\x3e`}, + // Escaping text spans. + {"<!--", `\x3c!\-\-`}, + {"-->", `\-\-\x3e`}, + {"*", `\*`}, + {"+", `\x2b`}, + {"?", `\?`}, + {"[](){}", `\[\]\(\)\{\}`}, + {"$foo|x.y", `\$foo\|x\.y`}, + {"x^y", `x\^y`}, + } + + for _, test := range tests { + esc := jsRegexpEscaper(test.x) + if esc != test.esc { + t.Errorf("%q: want %q got %q", test.x, test.esc, esc) + } + } +} + +func TestEscapersOnLower7AndSelectHighCodepoints(t *testing.T) { + input := ("\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./` + + `0123456789:;<=>?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\]^_` + + "`abcdefghijklmno" + + "pqrstuvwxyz{|}~\x7f" + + "\u00A0\u0100\u2028\u2029\ufeff\U0001D11E") + + tests := []struct { + name string + escaper func(...interface{}) string + escaped string + }{ + { + "jsStrEscaper", + jsStrEscaper, + "\\0\x01\x02\x03\x04\x05\x06\x07" + + "\x08\\t\\n\\x0b\\f\\r\x0E\x0F" + + "\x10\x11\x12\x13\x14\x15\x16\x17" + + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !\x22#$%\x26\x27()*\x2b,-.\/` + + `0123456789:;\x3c=\x3e?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\\]^_` + + "`abcdefghijklmno" + + "pqrstuvwxyz{|}~\x7f" + + "\u00A0\u0100\\u2028\\u2029\ufeff\U0001D11E", + }, + { + "jsRegexpEscaper", + jsRegexpEscaper, + "\\0\x01\x02\x03\x04\x05\x06\x07" + + "\x08\\t\\n\\x0b\\f\\r\x0E\x0F" + + "\x10\x11\x12\x13\x14\x15\x16\x17" + + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !\x22#\$%\x26\x27\(\)\*\x2b,\-\.\/` + + `0123456789:;\x3c=\x3e\?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ\[\\\]\^_` + + "`abcdefghijklmno" + + `pqrstuvwxyz\{\|\}~` + "\u007f" + + "\u00A0\u0100\\u2028\\u2029\ufeff\U0001D11E", + }, + } + + for _, test := range tests { + if s := test.escaper(input); s != test.escaped { + t.Errorf("%s once: want\n\t%q\ngot\n\t%q", test.name, test.escaped, s) + continue + } + + // Escape it rune by rune to make sure that any + // fast-path checking does not break escaping. + var buf bytes.Buffer + for _, c := range input { + buf.WriteString(test.escaper(string(c))) + } + + if s := buf.String(); s != test.escaped { + t.Errorf("%s rune-wise: want\n\t%q\ngot\n\t%q", test.name, test.escaped, s) + continue + } + } +} + +func BenchmarkJSValEscaperWithNum(b *testing.B) { + for i := 0; i < b.N; i++ { + jsValEscaper(3.141592654) + } +} + +func BenchmarkJSValEscaperWithStr(b *testing.B) { + for i := 0; i < b.N; i++ { + jsValEscaper("The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>") + } +} + +func BenchmarkJSValEscaperWithStrNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + jsValEscaper("The quick, brown fox jumps over the lazy dog") + } +} + +func BenchmarkJSValEscaperWithObj(b *testing.B) { + o := struct { + S string + N int + }{ + "The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>\u2028", + 42, + } + for i := 0; i < b.N; i++ { + jsValEscaper(o) + } +} + +func BenchmarkJSValEscaperWithObjNoSpecials(b *testing.B) { + o := struct { + S string + N int + }{ + "The quick, brown fox jumps over the lazy dog", + 42, + } + for i := 0; i < b.N; i++ { + jsValEscaper(o) + } +} + +func BenchmarkJSStrEscaperNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + jsStrEscaper("The quick, brown fox jumps over the lazy dog.") + } +} + +func BenchmarkJSStrEscaper(b *testing.B) { + for i := 0; i < b.N; i++ { + jsStrEscaper("The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>") + } +} + +func BenchmarkJSRegexpEscaperNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + jsRegexpEscaper("The quick, brown fox jumps over the lazy dog") + } +} + +func BenchmarkJSRegexpEscaper(b *testing.B) { + for i := 0; i < b.N; i++ { + jsRegexpEscaper("The <i>quick</i>,\r\n<span style='color:brown'>brown</span> fox jumps\u2028over the <canine class=\"lazy\">dog</canine>") + } +} diff --git a/libgo/go/exp/template/html/transition.go b/libgo/go/exp/template/html/transition.go new file mode 100644 index 0000000..49a1451 --- /dev/null +++ b/libgo/go/exp/template/html/transition.go @@ -0,0 +1,553 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "strings" +) + +// transitionFunc is the array of context transition functions for text nodes. +// A transition function takes a context and template text input, and returns +// the updated context and the number of bytes consumed from the front of the +// input. +var transitionFunc = [...]func(context, []byte) (context, int){ + stateText: tText, + stateTag: tTag, + stateAttrName: tAttrName, + stateAfterName: tAfterName, + stateBeforeValue: tBeforeValue, + stateHTMLCmt: tHTMLCmt, + stateRCDATA: tSpecialTagEnd, + stateAttr: tAttr, + stateURL: tURL, + stateJS: tJS, + stateJSDqStr: tJSDelimited, + stateJSSqStr: tJSDelimited, + stateJSRegexp: tJSDelimited, + stateJSBlockCmt: tBlockCmt, + stateJSLineCmt: tLineCmt, + stateCSS: tCSS, + stateCSSDqStr: tCSSStr, + stateCSSSqStr: tCSSStr, + stateCSSDqURL: tCSSStr, + stateCSSSqURL: tCSSStr, + stateCSSURL: tCSSStr, + stateCSSBlockCmt: tBlockCmt, + stateCSSLineCmt: tLineCmt, + stateError: tError, +} + +var commentStart = []byte("<!--") +var commentEnd = []byte("-->") + +// tText is the context transition function for the text state. +func tText(c context, s []byte) (context, int) { + k := 0 + for { + i := k + bytes.IndexByte(s[k:], '<') + if i < k || i+1 == len(s) { + return c, len(s) + } else if i+4 <= len(s) && bytes.Equal(commentStart, s[i:i+4]) { + return context{state: stateHTMLCmt}, i + 4 + } + i++ + end := false + if s[i] == '/' { + if i+1 == len(s) { + return c, len(s) + } + end, i = true, i+1 + } + j, e := eatTagName(s, i) + if j != i { + if end { + e = elementNone + } + // We've found an HTML tag. + return context{state: stateTag, element: e}, j + } + k = j + } + panic("unreachable") +} + +var elementContentType = [...]state{ + elementNone: stateText, + elementScript: stateJS, + elementStyle: stateCSS, + elementTextarea: stateRCDATA, + elementTitle: stateRCDATA, +} + +// tTag is the context transition function for the tag state. +func tTag(c context, s []byte) (context, int) { + // Find the attribute name. + i := eatWhiteSpace(s, 0) + if i == len(s) { + return c, len(s) + } + if s[i] == '>' { + return context{ + state: elementContentType[c.element], + element: c.element, + }, i + 1 + } + j, err := eatAttrName(s, i) + if err != nil { + return context{state: stateError, err: err}, len(s) + } + state, attr := stateTag, attrNone + if i == j { + return context{ + state: stateError, + err: errorf(ErrBadHTML, 0, "expected space, attr name, or end of tag, but got %q", s[i:]), + }, len(s) + } + switch attrType(string(s[i:j])) { + case contentTypeURL: + attr = attrURL + case contentTypeCSS: + attr = attrStyle + case contentTypeJS: + attr = attrScript + } + if j == len(s) { + state = stateAttrName + } else { + state = stateAfterName + } + return context{state: state, element: c.element, attr: attr}, j +} + +// tAttrName is the context transition function for stateAttrName. +func tAttrName(c context, s []byte) (context, int) { + i, err := eatAttrName(s, 0) + if err != nil { + return context{state: stateError, err: err}, len(s) + } else if i != len(s) { + c.state = stateAfterName + } + return c, i +} + +// tAfterName is the context transition function for stateAfterName. +func tAfterName(c context, s []byte) (context, int) { + // Look for the start of the value. + i := eatWhiteSpace(s, 0) + if i == len(s) { + return c, len(s) + } else if s[i] != '=' { + // Occurs due to tag ending '>', and valueless attribute. + c.state = stateTag + return c, i + } + c.state = stateBeforeValue + // Consume the "=". + return c, i + 1 +} + +var attrStartStates = [...]state{ + attrNone: stateAttr, + attrScript: stateJS, + attrStyle: stateCSS, + attrURL: stateURL, +} + +// tBeforeValue is the context transition function for stateBeforeValue. +func tBeforeValue(c context, s []byte) (context, int) { + i := eatWhiteSpace(s, 0) + if i == len(s) { + return c, len(s) + } + // Find the attribute delimiter. + delim := delimSpaceOrTagEnd + switch s[i] { + case '\'': + delim, i = delimSingleQuote, i+1 + case '"': + delim, i = delimDoubleQuote, i+1 + } + c.state, c.delim, c.attr = attrStartStates[c.attr], delim, attrNone + return c, i +} + +// tHTMLCmt is the context transition function for stateHTMLCmt. +func tHTMLCmt(c context, s []byte) (context, int) { + if i := bytes.Index(s, commentEnd); i != -1 { + return context{}, i + 3 + } + return c, len(s) +} + +// specialTagEndMarkers maps element types to the character sequence that +// case-insensitively signals the end of the special tag body. +var specialTagEndMarkers = [...]string{ + elementScript: "</script", + elementStyle: "</style", + elementTextarea: "</textarea", + elementTitle: "</title", +} + +// tSpecialTagEnd is the context transition function for raw text and RCDATA +// element states. +func tSpecialTagEnd(c context, s []byte) (context, int) { + if c.element != elementNone { + if i := strings.Index(strings.ToLower(string(s)), specialTagEndMarkers[c.element]); i != -1 { + return context{}, i + } + } + return c, len(s) +} + +// tAttr is the context transition function for the attribute state. +func tAttr(c context, s []byte) (context, int) { + return c, len(s) +} + +// tURL is the context transition function for the URL state. +func tURL(c context, s []byte) (context, int) { + if bytes.IndexAny(s, "#?") >= 0 { + c.urlPart = urlPartQueryOrFrag + } else if len(s) != eatWhiteSpace(s, 0) && c.urlPart == urlPartNone { + // HTML5 uses "Valid URL potentially surrounded by spaces" for + // attrs: http://www.w3.org/TR/html5/index.html#attributes-1 + c.urlPart = urlPartPreQuery + } + return c, len(s) +} + +// tJS is the context transition function for the JS state. +func tJS(c context, s []byte) (context, int) { + i := bytes.IndexAny(s, `"'/`) + if i == -1 { + // Entire input is non string, comment, regexp tokens. + c.jsCtx = nextJSCtx(s, c.jsCtx) + return c, len(s) + } + c.jsCtx = nextJSCtx(s[:i], c.jsCtx) + switch s[i] { + case '"': + c.state, c.jsCtx = stateJSDqStr, jsCtxRegexp + case '\'': + c.state, c.jsCtx = stateJSSqStr, jsCtxRegexp + case '/': + switch { + case i+1 < len(s) && s[i+1] == '/': + c.state, i = stateJSLineCmt, i+1 + case i+1 < len(s) && s[i+1] == '*': + c.state, i = stateJSBlockCmt, i+1 + case c.jsCtx == jsCtxRegexp: + c.state = stateJSRegexp + case c.jsCtx == jsCtxDivOp: + c.jsCtx = jsCtxRegexp + default: + return context{ + state: stateError, + err: errorf(ErrSlashAmbig, 0, "'/' could start a division or regexp: %.32q", s[i:]), + }, len(s) + } + default: + panic("unreachable") + } + return c, i + 1 +} + +// tJSDelimited is the context transition function for the JS string and regexp +// states. +func tJSDelimited(c context, s []byte) (context, int) { + specials := `\"` + switch c.state { + case stateJSSqStr: + specials = `\'` + case stateJSRegexp: + specials = `\/[]` + } + + k, inCharset := 0, false + for { + i := k + bytes.IndexAny(s[k:], specials) + if i < k { + break + } + switch s[i] { + case '\\': + i++ + if i == len(s) { + return context{ + state: stateError, + err: errorf(ErrPartialEscape, 0, "unfinished escape sequence in JS string: %q", s), + }, len(s) + } + case '[': + inCharset = true + case ']': + inCharset = false + default: + // end delimiter + if !inCharset { + c.state, c.jsCtx = stateJS, jsCtxDivOp + return c, i + 1 + } + } + k = i + 1 + } + + if inCharset { + // This can be fixed by making context richer if interpolation + // into charsets is desired. + return context{ + state: stateError, + err: errorf(ErrPartialCharset, 0, "unfinished JS regexp charset: %q", s), + }, len(s) + } + + return c, len(s) +} + +var blockCommentEnd = []byte("*/") + +// tBlockCmt is the context transition function for /*comment*/ states. +func tBlockCmt(c context, s []byte) (context, int) { + i := bytes.Index(s, blockCommentEnd) + if i == -1 { + return c, len(s) + } + switch c.state { + case stateJSBlockCmt: + c.state = stateJS + case stateCSSBlockCmt: + c.state = stateCSS + default: + panic(c.state.String()) + } + return c, i + 2 +} + +// tLineCmt is the context transition function for //comment states. +func tLineCmt(c context, s []byte) (context, int) { + var lineTerminators string + var endState state + switch c.state { + case stateJSLineCmt: + lineTerminators, endState = "\n\r\u2028\u2029", stateJS + case stateCSSLineCmt: + lineTerminators, endState = "\n\f\r", stateCSS + // Line comments are not part of any published CSS standard but + // are supported by the 4 major browsers. + // This defines line comments as + // LINECOMMENT ::= "//" [^\n\f\d]* + // since http://www.w3.org/TR/css3-syntax/#SUBTOK-nl defines + // newlines: + // nl ::= #xA | #xD #xA | #xD | #xC + default: + panic(c.state.String()) + } + + i := bytes.IndexAny(s, lineTerminators) + if i == -1 { + return c, len(s) + } + c.state = endState + // Per section 7.4 of EcmaScript 5 : http://es5.github.com/#x7.4 + // "However, the LineTerminator at the end of the line is not + // considered to be part of the single-line comment; it is + // recognized separately by the lexical grammar and becomes part + // of the stream of input elements for the syntactic grammar." + return c, i +} + +// tCSS is the context transition function for the CSS state. +func tCSS(c context, s []byte) (context, int) { + // CSS quoted strings are almost never used except for: + // (1) URLs as in background: "/foo.png" + // (2) Multiword font-names as in font-family: "Times New Roman" + // (3) List separators in content values as in inline-lists: + // <style> + // ul.inlineList { list-style: none; padding:0 } + // ul.inlineList > li { display: inline } + // ul.inlineList > li:before { content: ", " } + // ul.inlineList > li:first-child:before { content: "" } + // </style> + // <ul class=inlineList><li>One<li>Two<li>Three</ul> + // (4) Attribute value selectors as in a[href="http://example.com/"] + // + // We conservatively treat all strings as URLs, but make some + // allowances to avoid confusion. + // + // In (1), our conservative assumption is justified. + // In (2), valid font names do not contain ':', '?', or '#', so our + // conservative assumption is fine since we will never transition past + // urlPartPreQuery. + // In (3), our protocol heuristic should not be tripped, and there + // should not be non-space content after a '?' or '#', so as long as + // we only %-encode RFC 3986 reserved characters we are ok. + // In (4), we should URL escape for URL attributes, and for others we + // have the attribute name available if our conservative assumption + // proves problematic for real code. + + k := 0 + for { + i := k + bytes.IndexAny(s[k:], `("'/`) + if i < k { + return c, len(s) + } + switch s[i] { + case '(': + // Look for url to the left. + p := bytes.TrimRight(s[:i], "\t\n\f\r ") + if endsWithCSSKeyword(p, "url") { + j := len(s) - len(bytes.TrimLeft(s[i+1:], "\t\n\f\r ")) + switch { + case j != len(s) && s[j] == '"': + c.state, j = stateCSSDqURL, j+1 + case j != len(s) && s[j] == '\'': + c.state, j = stateCSSSqURL, j+1 + default: + c.state = stateCSSURL + } + return c, j + } + case '/': + if i+1 < len(s) { + switch s[i+1] { + case '/': + c.state = stateCSSLineCmt + return c, i + 2 + case '*': + c.state = stateCSSBlockCmt + return c, i + 2 + } + } + case '"': + c.state = stateCSSDqStr + return c, i + 1 + case '\'': + c.state = stateCSSSqStr + return c, i + 1 + } + k = i + 1 + } + panic("unreachable") +} + +// tCSSStr is the context transition function for the CSS string and URL states. +func tCSSStr(c context, s []byte) (context, int) { + var endAndEsc string + switch c.state { + case stateCSSDqStr, stateCSSDqURL: + endAndEsc = `\"` + case stateCSSSqStr, stateCSSSqURL: + endAndEsc = `\'` + case stateCSSURL: + // Unquoted URLs end with a newline or close parenthesis. + // The below includes the wc (whitespace character) and nl. + endAndEsc = "\\\t\n\f\r )" + default: + panic(c.state.String()) + } + + k := 0 + for { + i := k + bytes.IndexAny(s[k:], endAndEsc) + if i < k { + c, nread := tURL(c, decodeCSS(s[k:])) + return c, k + nread + } + if s[i] == '\\' { + i++ + if i == len(s) { + return context{ + state: stateError, + err: errorf(ErrPartialEscape, 0, "unfinished escape sequence in CSS string: %q", s), + }, len(s) + } + } else { + c.state = stateCSS + return c, i + 1 + } + c, _ = tURL(c, decodeCSS(s[:i+1])) + k = i + 1 + } + panic("unreachable") +} + +// tError is the context transition function for the error state. +func tError(c context, s []byte) (context, int) { + return c, len(s) +} + +// eatAttrName returns the largest j such that s[i:j] is an attribute name. +// It returns an error if s[i:] does not look like it begins with an +// attribute name, such as encountering a quote mark without a preceding +// equals sign. +func eatAttrName(s []byte, i int) (int, *Error) { + for j := i; j < len(s); j++ { + switch s[j] { + case ' ', '\t', '\n', '\f', '\r', '=', '>': + return j, nil + case '\'', '"', '<': + // These result in a parse warning in HTML5 and are + // indicative of serious problems if seen in an attr + // name in a template. + return -1, errorf(ErrBadHTML, 0, "%q in attribute name: %.32q", s[j:j+1], s) + default: + // No-op. + } + } + return len(s), nil +} + +var elementNameMap = map[string]element{ + "script": elementScript, + "style": elementStyle, + "textarea": elementTextarea, + "title": elementTitle, +} + +// asciiAlpha returns whether c is an ASCII letter. +func asciiAlpha(c byte) bool { + return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' +} + +// asciiAlphaNum returns whether c is an ASCII letter or digit. +func asciiAlphaNum(c byte) bool { + return asciiAlpha(c) || '0' <= c && c <= '9' +} + +// eatTagName returns the largest j such that s[i:j] is a tag name and the tag type. +func eatTagName(s []byte, i int) (int, element) { + if i == len(s) || !asciiAlpha(s[i]) { + return i, elementNone + } + j := i + 1 + for j < len(s) { + x := s[j] + if asciiAlphaNum(x) { + j++ + continue + } + // Allow "x-y" or "x:y" but not "x-", "-y", or "x--y". + if (x == ':' || x == '-') && j+1 < len(s) && asciiAlphaNum(s[j+1]) { + j += 2 + continue + } + break + } + return j, elementNameMap[strings.ToLower(string(s[i:j]))] +} + +// eatWhiteSpace returns the largest j such that s[i:j] is white space. +func eatWhiteSpace(s []byte, i int) int { + for j := i; j < len(s); j++ { + switch s[j] { + case ' ', '\t', '\n', '\f', '\r': + // No-op. + default: + return j + } + } + return len(s) +} diff --git a/libgo/go/exp/template/html/url.go b/libgo/go/exp/template/html/url.go new file mode 100644 index 0000000..5b19df0 --- /dev/null +++ b/libgo/go/exp/template/html/url.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "fmt" + "strings" +) + +// urlFilter returns its input unless it contains an unsafe protocol in which +// case it defangs the entire URL. +func urlFilter(args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeURL { + return s + } + if i := strings.IndexRune(s, ':'); i >= 0 && strings.IndexRune(s[:i], '/') < 0 { + protocol := strings.ToLower(s[:i]) + if protocol != "http" && protocol != "https" && protocol != "mailto" { + return "#" + filterFailsafe + } + } + return s +} + +// urlEscaper produces an output that can be embedded in a URL query. +// The output can be embedded in an HTML attribute without further escaping. +func urlEscaper(args ...interface{}) string { + return urlProcessor(false, args...) +} + +// urlEscaper normalizes URL content so it can be embedded in a quote-delimited +// string or parenthesis delimited url(...). +// The normalizer does not encode all HTML specials. Specifically, it does not +// encode '&' so correct embedding in an HTML attribute requires escaping of +// '&' to '&'. +func urlNormalizer(args ...interface{}) string { + return urlProcessor(true, args...) +} + +// urlProcessor normalizes (when norm is true) or escapes its input to produce +// a valid hierarchical or opaque URL part. +func urlProcessor(norm bool, args ...interface{}) string { + s, t := stringify(args...) + if t == contentTypeURL { + norm = true + } + var b bytes.Buffer + written := 0 + // The byte loop below assumes that all URLs use UTF-8 as the + // content-encoding. This is similar to the URI to IRI encoding scheme + // defined in section 3.1 of RFC 3987, and behaves the same as the + // EcmaScript builtin encodeURIComponent. + // It should not cause any misencoding of URLs in pages with + // Content-type: text/html;charset=UTF-8. + for i, n := 0, len(s); i < n; i++ { + c := s[i] + switch c { + // Single quote and parens are sub-delims in RFC 3986, but we + // escape them so the output can be embedded in in single + // quoted attributes and unquoted CSS url(...) constructs. + // Single quotes are reserved in URLs, but are only used in + // the obsolete "mark" rule in an appendix in RFC 3986 + // so can be safely encoded. + case '!', '#', '$', '&', '*', '+', ',', '/', ':', ';', '=', '?', '@', '[', ']': + if norm { + continue + } + // Unreserved according to RFC 3986 sec 2.3 + // "For consistency, percent-encoded octets in the ranges of + // ALPHA (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), + // period (%2E), underscore (%5F), or tilde (%7E) should not be + // created by URI producers + case '-', '.', '_', '~': + continue + case '%': + // When normalizing do not re-encode valid escapes. + if norm && i+2 < len(s) && isHex(s[i+1]) && isHex(s[i+2]) { + continue + } + default: + // Unreserved according to RFC 3986 sec 2.3 + if 'a' <= c && c <= 'z' { + continue + } + if 'A' <= c && c <= 'Z' { + continue + } + if '0' <= c && c <= '9' { + continue + } + } + b.WriteString(s[written:i]) + fmt.Fprintf(&b, "%%%02x", c) + written = i + 1 + } + if written == 0 { + return s + } + b.WriteString(s[written:]) + return b.String() +} diff --git a/libgo/go/exp/template/html/url_test.go b/libgo/go/exp/template/html/url_test.go new file mode 100644 index 0000000..b846231 --- /dev/null +++ b/libgo/go/exp/template/html/url_test.go @@ -0,0 +1,112 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "testing" +) + +func TestURLNormalizer(t *testing.T) { + tests := []struct { + url, want string + }{ + {"", ""}, + { + "http://example.com:80/foo/bar?q=foo%20&bar=x+y#frag", + "http://example.com:80/foo/bar?q=foo%20&bar=x+y#frag", + }, + {" ", "%20"}, + {"%7c", "%7c"}, + {"%7C", "%7C"}, + {"%2", "%252"}, + {"%", "%25"}, + {"%z", "%25z"}, + {"/foo|bar/%5c\u1234", "/foo%7cbar/%5c%e1%88%b4"}, + } + for _, test := range tests { + if got := urlNormalizer(test.url); test.want != got { + t.Errorf("%q: want\n\t%q\nbut got\n\t%q", test.url, test.want, got) + } + if test.want != urlNormalizer(test.want) { + t.Errorf("not idempotent: %q", test.want) + } + } +} + +func TestURLFilters(t *testing.T) { + input := ("\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./` + + `0123456789:;<=>?` + + `@ABCDEFGHIJKLMNO` + + `PQRSTUVWXYZ[\]^_` + + "`abcdefghijklmno" + + "pqrstuvwxyz{|}~\x7f" + + "\u00A0\u0100\u2028\u2029\ufeff\U0001D11E") + + tests := []struct { + name string + escaper func(...interface{}) string + escaped string + }{ + { + "urlEscaper", + urlEscaper, + "%00%01%02%03%04%05%06%07%08%09%0a%0b%0c%0d%0e%0f" + + "%10%11%12%13%14%15%16%17%18%19%1a%1b%1c%1d%1e%1f" + + "%20%21%22%23%24%25%26%27%28%29%2a%2b%2c-.%2f" + + "0123456789%3a%3b%3c%3d%3e%3f" + + "%40ABCDEFGHIJKLMNO" + + "PQRSTUVWXYZ%5b%5c%5d%5e_" + + "%60abcdefghijklmno" + + "pqrstuvwxyz%7b%7c%7d~%7f" + + "%c2%a0%c4%80%e2%80%a8%e2%80%a9%ef%bb%bf%f0%9d%84%9e", + }, + { + "urlNormalizer", + urlNormalizer, + "%00%01%02%03%04%05%06%07%08%09%0a%0b%0c%0d%0e%0f" + + "%10%11%12%13%14%15%16%17%18%19%1a%1b%1c%1d%1e%1f" + + "%20!%22#$%25&%27%28%29*+,-./" + + "0123456789:;%3c=%3e?" + + "@ABCDEFGHIJKLMNO" + + "PQRSTUVWXYZ[%5c]%5e_" + + "%60abcdefghijklmno" + + "pqrstuvwxyz%7b%7c%7d~%7f" + + "%c2%a0%c4%80%e2%80%a8%e2%80%a9%ef%bb%bf%f0%9d%84%9e", + }, + } + + for _, test := range tests { + if s := test.escaper(input); s != test.escaped { + t.Errorf("%s: want\n\t%q\ngot\n\t%q", test.name, test.escaped, s) + continue + } + } +} + +func BenchmarkURLEscaper(b *testing.B) { + for i := 0; i < b.N; i++ { + urlEscaper("http://example.com:80/foo?q=bar%20&baz=x+y#frag") + } +} + +func BenchmarkURLEscaperNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + urlEscaper("TheQuickBrownFoxJumpsOverTheLazyDog.") + } +} + +func BenchmarkURLNormalizer(b *testing.B) { + for i := 0; i < b.N; i++ { + urlNormalizer("The quick brown fox jumps over the lazy dog.\n") + } +} + +func BenchmarkURLNormalizerNoSpecials(b *testing.B) { + for i := 0; i < b.N; i++ { + urlNormalizer("http://example.com:80/foo?q=bar%20&baz=x+y#frag") + } +} diff --git a/libgo/go/exp/terminal/shell.go b/libgo/go/exp/terminal/shell.go new file mode 100644 index 0000000..e3f5847 --- /dev/null +++ b/libgo/go/exp/terminal/shell.go @@ -0,0 +1,359 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package terminal + +import ( + "os" + "io" +) + +// Shell contains the state for running a VT100 terminal that is capable of +// reading lines of input. +type Shell struct { + c io.ReadWriter + prompt string + + // line is the current line being entered. + line []byte + // pos is the logical position of the cursor in line + pos int + + // cursorX contains the current X value of the cursor where the left + // edge is 0. cursorY contains the row number where the first row of + // the current line is 0. + cursorX, cursorY int + // maxLine is the greatest value of cursorY so far. + maxLine int + + termWidth, termHeight int + + // outBuf contains the terminal data to be sent. + outBuf []byte + // remainder contains the remainder of any partial key sequences after + // a read. It aliases into inBuf. + remainder []byte + inBuf [256]byte +} + +// NewShell runs a VT100 terminal on the given ReadWriter. If the ReadWriter is +// a local terminal, that terminal must first have been put into raw mode. +// prompt is a string that is written at the start of each input line (i.e. +// "> "). +func NewShell(c io.ReadWriter, prompt string) *Shell { + return &Shell{ + c: c, + prompt: prompt, + termWidth: 80, + termHeight: 24, + } +} + +const ( + keyCtrlD = 4 + keyEnter = '\r' + keyEscape = 27 + keyBackspace = 127 + keyUnknown = 256 + iota + keyUp + keyDown + keyLeft + keyRight + keyAltLeft + keyAltRight +) + +// bytesToKey tries to parse a key sequence from b. If successful, it returns +// the key and the remainder of the input. Otherwise it returns -1. +func bytesToKey(b []byte) (int, []byte) { + if len(b) == 0 { + return -1, nil + } + + if b[0] != keyEscape { + return int(b[0]), b[1:] + } + + if len(b) >= 3 && b[0] == keyEscape && b[1] == '[' { + switch b[2] { + case 'A': + return keyUp, b[3:] + case 'B': + return keyDown, b[3:] + case 'C': + return keyRight, b[3:] + case 'D': + return keyLeft, b[3:] + } + } + + if len(b) >= 6 && b[0] == keyEscape && b[1] == '[' && b[2] == '1' && b[3] == ';' && b[4] == '3' { + switch b[5] { + case 'C': + return keyAltRight, b[6:] + case 'D': + return keyAltLeft, b[6:] + } + } + + // If we get here then we have a key that we don't recognise, or a + // partial sequence. It's not clear how one should find the end of a + // sequence without knowing them all, but it seems that [a-zA-Z] only + // appears at the end of a sequence. + for i, c := range b[0:] { + if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' { + return keyUnknown, b[i+1:] + } + } + + return -1, b +} + +// queue appends data to the end of ss.outBuf +func (ss *Shell) queue(data []byte) { + if len(ss.outBuf)+len(data) > cap(ss.outBuf) { + newOutBuf := make([]byte, len(ss.outBuf), 2*(len(ss.outBuf)+len(data))) + copy(newOutBuf, ss.outBuf) + ss.outBuf = newOutBuf + } + + oldLen := len(ss.outBuf) + ss.outBuf = ss.outBuf[:len(ss.outBuf)+len(data)] + copy(ss.outBuf[oldLen:], data) +} + +var eraseUnderCursor = []byte{' ', keyEscape, '[', 'D'} + +func isPrintable(key int) bool { + return key >= 32 && key < 127 +} + +// moveCursorToPos appends data to ss.outBuf which will move the cursor to the +// given, logical position in the text. +func (ss *Shell) moveCursorToPos(pos int) { + x := len(ss.prompt) + pos + y := x / ss.termWidth + x = x % ss.termWidth + + up := 0 + if y < ss.cursorY { + up = ss.cursorY - y + } + + down := 0 + if y > ss.cursorY { + down = y - ss.cursorY + } + + left := 0 + if x < ss.cursorX { + left = ss.cursorX - x + } + + right := 0 + if x > ss.cursorX { + right = x - ss.cursorX + } + + movement := make([]byte, 3*(up+down+left+right)) + m := movement + for i := 0; i < up; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'A' + m = m[3:] + } + for i := 0; i < down; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'B' + m = m[3:] + } + for i := 0; i < left; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'D' + m = m[3:] + } + for i := 0; i < right; i++ { + m[0] = keyEscape + m[1] = '[' + m[2] = 'C' + m = m[3:] + } + + ss.cursorX = x + ss.cursorY = y + ss.queue(movement) +} + +const maxLineLength = 4096 + +// handleKey processes the given key and, optionally, returns a line of text +// that the user has entered. +func (ss *Shell) handleKey(key int) (line string, ok bool) { + switch key { + case keyBackspace: + if ss.pos == 0 { + return + } + ss.pos-- + + copy(ss.line[ss.pos:], ss.line[1+ss.pos:]) + ss.line = ss.line[:len(ss.line)-1] + ss.writeLine(ss.line[ss.pos:]) + ss.moveCursorToPos(ss.pos) + ss.queue(eraseUnderCursor) + case keyAltLeft: + // move left by a word. + if ss.pos == 0 { + return + } + ss.pos-- + for ss.pos > 0 { + if ss.line[ss.pos] != ' ' { + break + } + ss.pos-- + } + for ss.pos > 0 { + if ss.line[ss.pos] == ' ' { + ss.pos++ + break + } + ss.pos-- + } + ss.moveCursorToPos(ss.pos) + case keyAltRight: + // move right by a word. + for ss.pos < len(ss.line) { + if ss.line[ss.pos] == ' ' { + break + } + ss.pos++ + } + for ss.pos < len(ss.line) { + if ss.line[ss.pos] != ' ' { + break + } + ss.pos++ + } + ss.moveCursorToPos(ss.pos) + case keyLeft: + if ss.pos == 0 { + return + } + ss.pos-- + ss.moveCursorToPos(ss.pos) + case keyRight: + if ss.pos == len(ss.line) { + return + } + ss.pos++ + ss.moveCursorToPos(ss.pos) + case keyEnter: + ss.moveCursorToPos(len(ss.line)) + ss.queue([]byte("\r\n")) + line = string(ss.line) + ok = true + ss.line = ss.line[:0] + ss.pos = 0 + ss.cursorX = 0 + ss.cursorY = 0 + ss.maxLine = 0 + default: + if !isPrintable(key) { + return + } + if len(ss.line) == maxLineLength { + return + } + if len(ss.line) == cap(ss.line) { + newLine := make([]byte, len(ss.line), 2*(1+len(ss.line))) + copy(newLine, ss.line) + ss.line = newLine + } + ss.line = ss.line[:len(ss.line)+1] + copy(ss.line[ss.pos+1:], ss.line[ss.pos:]) + ss.line[ss.pos] = byte(key) + ss.writeLine(ss.line[ss.pos:]) + ss.pos++ + ss.moveCursorToPos(ss.pos) + } + return +} + +func (ss *Shell) writeLine(line []byte) { + for len(line) != 0 { + if ss.cursorX == ss.termWidth { + ss.queue([]byte("\r\n")) + ss.cursorX = 0 + ss.cursorY++ + if ss.cursorY > ss.maxLine { + ss.maxLine = ss.cursorY + } + } + + remainingOnLine := ss.termWidth - ss.cursorX + todo := len(line) + if todo > remainingOnLine { + todo = remainingOnLine + } + ss.queue(line[:todo]) + ss.cursorX += todo + line = line[todo:] + } +} + +func (ss *Shell) Write(buf []byte) (n int, err os.Error) { + return ss.c.Write(buf) +} + +// ReadLine returns a line of input from the terminal. +func (ss *Shell) ReadLine() (line string, err os.Error) { + ss.writeLine([]byte(ss.prompt)) + ss.c.Write(ss.outBuf) + ss.outBuf = ss.outBuf[:0] + + for { + // ss.remainder is a slice at the beginning of ss.inBuf + // containing a partial key sequence + readBuf := ss.inBuf[len(ss.remainder):] + var n int + n, err = ss.c.Read(readBuf) + if err != nil { + return + } + + if err == nil { + ss.remainder = ss.inBuf[:n+len(ss.remainder)] + rest := ss.remainder + lineOk := false + for !lineOk { + var key int + key, rest = bytesToKey(rest) + if key < 0 { + break + } + if key == keyCtrlD { + return "", os.EOF + } + line, lineOk = ss.handleKey(key) + } + if len(rest) > 0 { + n := copy(ss.inBuf[:], rest) + ss.remainder = ss.inBuf[:n] + } else { + ss.remainder = nil + } + ss.c.Write(ss.outBuf) + ss.outBuf = ss.outBuf[:0] + if lineOk { + return + } + continue + } + } + panic("unreachable") +} diff --git a/libgo/go/exp/terminal/shell_test.go b/libgo/go/exp/terminal/shell_test.go new file mode 100644 index 0000000..2bbe4a4 --- /dev/null +++ b/libgo/go/exp/terminal/shell_test.go @@ -0,0 +1,110 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package terminal + +import ( + "testing" + "os" +) + +type MockTerminal struct { + toSend []byte + bytesPerRead int + received []byte +} + +func (c *MockTerminal) Read(data []byte) (n int, err os.Error) { + n = len(data) + if n == 0 { + return + } + if n > len(c.toSend) { + n = len(c.toSend) + } + if n == 0 { + return 0, os.EOF + } + if c.bytesPerRead > 0 && n > c.bytesPerRead { + n = c.bytesPerRead + } + copy(data, c.toSend[:n]) + c.toSend = c.toSend[n:] + return +} + +func (c *MockTerminal) Write(data []byte) (n int, err os.Error) { + c.received = append(c.received, data...) + return len(data), nil +} + +func TestClose(t *testing.T) { + c := &MockTerminal{} + ss := NewShell(c, "> ") + line, err := ss.ReadLine() + if line != "" { + t.Errorf("Expected empty line but got: %s", line) + } + if err != os.EOF { + t.Errorf("Error should have been EOF but got: %s", err) + } +} + +var keyPressTests = []struct { + in string + line string + err os.Error +}{ + { + "", + "", + os.EOF, + }, + { + "\r", + "", + nil, + }, + { + "foo\r", + "foo", + nil, + }, + { + "a\x1b[Cb\r", // right + "ab", + nil, + }, + { + "a\x1b[Db\r", // left + "ba", + nil, + }, + { + "a\177b\r", // backspace + "b", + nil, + }, +} + +func TestKeyPresses(t *testing.T) { + for i, test := range keyPressTests { + for j := 0; j < len(test.in); j++ { + c := &MockTerminal{ + toSend: []byte(test.in), + bytesPerRead: j, + } + ss := NewShell(c, "> ") + line, err := ss.ReadLine() + if line != test.line { + t.Errorf("Line resulting from test %d (%d bytes per read) was '%s', expected '%s'", i, j, line, test.line) + break + } + if err != test.err { + t.Errorf("Error resulting from test %d (%d bytes per read) was '%v', expected '%v'", i, j, err, test.err) + break + } + } + } +} diff --git a/libgo/go/exp/terminal/terminal.go b/libgo/go/exp/terminal/terminal.go new file mode 100644 index 0000000..aacd909 --- /dev/null +++ b/libgo/go/exp/terminal/terminal.go @@ -0,0 +1,103 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package terminal provides support functions for dealing with terminals, as +// commonly found on UNIX systems. +// +// Putting a terminal into raw mode is the most common requirement: +// +// oldState, err := terminal.MakeRaw(0) +// if err != nil { +// panic(err.String()) +// } +// defer terminal.Restore(0, oldState) +package terminal + +import ( + "os" + "syscall" + "unsafe" +) + +// State contains the state of a terminal. +type State struct { + termios syscall.Termios +} + +// IsTerminal returns true if the given file descriptor is a terminal. +func IsTerminal(fd int) bool { + var termios syscall.Termios + _, _, e := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCGETS), uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return e == 0 +} + +// MakeRaw put the terminal connected to the given file descriptor into raw +// mode and returns the previous state of the terminal so that it can be +// restored. +func MakeRaw(fd int) (*State, os.Error) { + var oldState State + if _, _, e := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCGETS), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); e != 0 { + return nil, os.Errno(e) + } + + newState := oldState.termios + newState.Iflag &^= syscall.ISTRIP | syscall.INLCR | syscall.ICRNL | syscall.IGNCR | syscall.IXON | syscall.IXOFF + newState.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.ISIG + if _, _, e := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCSETS), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); e != 0 { + return nil, os.Errno(e) + } + + return &oldState, nil +} + +// Restore restores the terminal connected to the given file descriptor to a +// previous state. +func Restore(fd int, state *State) os.Error { + _, _, e := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCSETS), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) + return os.Errno(e) +} + +// ReadPassword reads a line of input from a terminal without local echo. This +// is commonly used for inputting passwords and other sensitive data. The slice +// returned does not include the \n. +func ReadPassword(fd int) ([]byte, os.Error) { + var oldState syscall.Termios + if _, _, e := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCGETS), uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); e != 0 { + return nil, os.Errno(e) + } + + newState := oldState + newState.Lflag &^= syscall.ECHO + if _, _, e := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCSETS), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); e != 0 { + return nil, os.Errno(e) + } + + defer func() { + syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TCSETS), uintptr(unsafe.Pointer(&oldState)), 0, 0, 0) + }() + + var buf [16]byte + var ret []byte + for { + n, errno := syscall.Read(fd, buf[:]) + if errno != 0 { + return nil, os.Errno(errno) + } + if n == 0 { + if len(ret) == 0 { + return nil, os.EOF + } + break + } + if buf[n-1] == '\n' { + n-- + } + ret = append(ret, buf[:n]...) + if n < len(buf) { + break + } + } + + return ret, nil +} diff --git a/libgo/go/go/types/check.go b/libgo/go/exp/types/check.go index 87e3e93..87e3e93 100644 --- a/libgo/go/go/types/check.go +++ b/libgo/go/exp/types/check.go diff --git a/libgo/go/go/types/check_test.go b/libgo/go/exp/types/check_test.go index 8be653f..034acd0 100644 --- a/libgo/go/go/types/check_test.go +++ b/libgo/go/exp/types/check_test.go @@ -154,7 +154,7 @@ func eliminate(t *testing.T, expected map[token.Pos]string, errors os.Error) { continue } // we have a match - eliminate this error - expected[pos] = "", false + delete(expected, pos) } else { // To keep in mind when analyzing failed test output: // If the same error position occurs multiple times in errors, diff --git a/libgo/go/go/types/const.go b/libgo/go/exp/types/const.go index 1ef95d9..1ef95d9 100644 --- a/libgo/go/go/types/const.go +++ b/libgo/go/exp/types/const.go diff --git a/libgo/go/go/types/exportdata.go b/libgo/go/exp/types/exportdata.go index 3835203..3835203 100644 --- a/libgo/go/go/types/exportdata.go +++ b/libgo/go/exp/types/exportdata.go diff --git a/libgo/go/go/types/gcimporter.go b/libgo/go/exp/types/gcimporter.go index 6ab1806..fe90f91 100644 --- a/libgo/go/go/types/gcimporter.go +++ b/libgo/go/exp/types/gcimporter.go @@ -142,6 +142,34 @@ func GcImporter(imports map[string]*ast.Object, path string) (pkg *ast.Object, e return } +// Declare inserts a named object of the given kind in scope. +func (p *gcParser) declare(scope *ast.Scope, kind ast.ObjKind, name string) *ast.Object { + // a type may have been declared before - if it exists + // already in the respective package scope, return that + // type + if kind == ast.Typ { + if obj := scope.Lookup(name); obj != nil { + assert(obj.Kind == ast.Typ) + return obj + } + } + + // any other object must be a newly declared object - + // create it and insert it into the package scope + obj := ast.NewObj(kind, name) + if scope.Insert(obj) != nil { + p.errorf("already declared: %v %s", kind, obj.Name) + } + + // a new type object is a named type and may be referred + // to before the underlying type is known - set it up + if kind == ast.Typ { + obj.Type = &Name{Obj: obj} + } + + return obj +} + // ---------------------------------------------------------------------------- // Error handling @@ -245,38 +273,14 @@ func (p *gcParser) parseDotIdent() string { return ident } -// ExportedName = ImportPath "." dotIdentifier . +// ExportedName = "@" ImportPath "." dotIdentifier . // -func (p *gcParser) parseExportedName(kind ast.ObjKind) *ast.Object { +func (p *gcParser) parseExportedName() (*ast.Object, string) { + p.expect('@') pkg := p.parsePkgId() p.expect('.') name := p.parseDotIdent() - - // a type may have been declared before - if it exists - // already in the respective package scope, return that - // type - scope := pkg.Data.(*ast.Scope) - if kind == ast.Typ { - if obj := scope.Lookup(name); obj != nil { - assert(obj.Kind == ast.Typ) - return obj - } - } - - // any other object must be a newly declared object - - // create it and insert it into the package scope - obj := ast.NewObj(kind, name) - if scope.Insert(obj) != nil { - p.errorf("already declared: %s", obj.Name) - } - - // a new type object is a named type and may be referred - // to before the underlying type is known - set it up - if kind == ast.Typ { - obj.Type = &Name{Obj: obj} - } - - return obj + return pkg, name } // ---------------------------------------------------------------------------- @@ -333,7 +337,7 @@ func (p *gcParser) parseName() (name string) { return } -// Field = Name Type [ ":" string_lit ] . +// Field = Name Type [ string_lit ] . // func (p *gcParser) parseField() (fld *ast.Object, tag string) { name := p.parseName() @@ -344,8 +348,7 @@ func (p *gcParser) parseField() (fld *ast.Object, tag string) { p.errorf("anonymous field expected") } } - if p.tok == ':' { - p.next() + if p.tok == scanner.String { tag = p.expect(scanner.String) } fld = ast.NewObj(ast.Var, name) @@ -380,7 +383,7 @@ func (p *gcParser) parseStructType() Type { return &Struct{Fields: fields, Tags: tags} } -// Parameter = ( identifier | "?" ) [ "..." ] Type [ ":" string_lit ] . +// Parameter = ( identifier | "?" ) [ "..." ] Type [ string_lit ] . // func (p *gcParser) parseParameter() (par *ast.Object, isVariadic bool) { name := p.parseName() @@ -393,8 +396,7 @@ func (p *gcParser) parseParameter() (par *ast.Object, isVariadic bool) { } ptyp := p.parseType() // ignore argument tag - if p.tok == ':' { - p.next() + if p.tok == scanner.String { p.expect(scanner.String) } par = ast.NewObj(ast.Var, name) @@ -439,7 +441,7 @@ func (p *gcParser) parseSignature() *Func { // optional result type var results []*ast.Object switch p.tok { - case scanner.Ident, scanner.String, '[', '*', '<': + case scanner.Ident, '[', '*', '<', '@': // single, unnamed result result := ast.NewObj(ast.Var, "_") result.Type = p.parseType() @@ -456,16 +458,13 @@ func (p *gcParser) parseSignature() *Func { return &Func{Params: params, Results: results, IsVariadic: isVariadic} } -// MethodSpec = identifier Signature . +// MethodSpec = ( identifier | ExportedName ) Signature . // func (p *gcParser) parseMethodSpec() *ast.Object { if p.tok == scanner.Ident { p.expect(scanner.Ident) } else { - // TODO(gri) should this be parseExportedName here? - p.parsePkgId() - p.expect('.') - p.parseDotIdent() + p.parseExportedName() } p.parseSignature() @@ -547,9 +546,10 @@ func (p *gcParser) parseType() Type { case "chan": return p.parseChanType() } - case scanner.String: + case '@': // TypeName - return p.parseExportedName(ast.Typ).Type.(Type) + pkg, name := p.parseExportedName() + return p.declare(pkg.Data.(*ast.Scope), ast.Typ, name).Type.(Type) case '[': p.next() // look ahead if p.tok == ']' { @@ -643,7 +643,8 @@ func (p *gcParser) parseNumber() Const { // func (p *gcParser) parseConstDecl() { p.expectKeyword("const") - obj := p.parseExportedName(ast.Con) + pkg, name := p.parseExportedName() + obj := p.declare(pkg.Data.(*ast.Scope), ast.Con, name) var x Const var typ Type if p.tok != '=' { @@ -693,7 +694,8 @@ func (p *gcParser) parseConstDecl() { // func (p *gcParser) parseTypeDecl() { p.expectKeyword("type") - obj := p.parseExportedName(ast.Typ) + pkg, name := p.parseExportedName() + obj := p.declare(pkg.Data.(*ast.Scope), ast.Typ, name) // The type object may have been imported before and thus already // have a type associated with it. We still need to parse the type @@ -712,20 +714,39 @@ func (p *gcParser) parseTypeDecl() { // func (p *gcParser) parseVarDecl() { p.expectKeyword("var") - obj := p.parseExportedName(ast.Var) + pkg, name := p.parseExportedName() + obj := p.declare(pkg.Data.(*ast.Scope), ast.Var, name) obj.Type = p.parseType() } -// FuncDecl = "func" ExportedName Signature . +// FuncBody = "{" ... "}" . +// +func (p *gcParser) parseFuncBody() { + p.expect('{') + for i := 1; i > 0; p.next() { + switch p.tok { + case '{': + i++ + case '}': + i-- + } + } +} + +// FuncDecl = "func" ExportedName Signature [ FuncBody ] . // func (p *gcParser) parseFuncDecl() { // "func" already consumed - obj := p.parseExportedName(ast.Fun) + pkg, name := p.parseExportedName() + obj := p.declare(pkg.Data.(*ast.Scope), ast.Fun, name) obj.Type = p.parseSignature() + if p.tok == '{' { + p.parseFuncBody() + } } // MethodDecl = "func" Receiver identifier Signature . -// Receiver = "(" ( identifier | "?" ) [ "*" ] ExportedName ")" . +// Receiver = "(" ( identifier | "?" ) [ "*" ] ExportedName ")" [ FuncBody ]. // func (p *gcParser) parseMethodDecl() { // "func" already consumed @@ -734,6 +755,9 @@ func (p *gcParser) parseMethodDecl() { p.expect(')') p.expect(scanner.Ident) p.parseSignature() + if p.tok == '{' { + p.parseFuncBody() + } } // Decl = [ ImportDecl | ConstDecl | TypeDecl | VarDecl | FuncDecl | MethodDecl ] "\n" . diff --git a/libgo/go/go/types/gcimporter_test.go b/libgo/go/exp/types/gcimporter_test.go index ec87f5d..ec87f5d 100644 --- a/libgo/go/go/types/gcimporter_test.go +++ b/libgo/go/exp/types/gcimporter_test.go diff --git a/libgo/go/go/types/testdata/exports.go b/libgo/go/exp/types/testdata/exports.go index ed63bf9..ed63bf9 100644 --- a/libgo/go/go/types/testdata/exports.go +++ b/libgo/go/exp/types/testdata/exports.go diff --git a/libgo/go/go/types/types.go b/libgo/go/exp/types/types.go index 3aa8968..3aa8968 100644 --- a/libgo/go/go/types/types.go +++ b/libgo/go/exp/types/types.go diff --git a/libgo/go/go/types/universe.go b/libgo/go/exp/types/universe.go index 6ae88e5..80db127 100644 --- a/libgo/go/go/types/universe.go +++ b/libgo/go/exp/types/universe.go @@ -81,6 +81,7 @@ func init() { defFun("close") defFun("complex") defFun("copy") + defFun("delete") defFun("imag") defFun("len") defFun("make") diff --git a/libgo/go/exp/winfsnotify/winfsnotify.go b/libgo/go/exp/winfsnotify/winfsnotify.go new file mode 100644 index 0000000..c5dfe99 --- /dev/null +++ b/libgo/go/exp/winfsnotify/winfsnotify.go @@ -0,0 +1,569 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package winfsnotify allows the user to receive +// file system event notifications on Windows. +package winfsnotify + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "syscall" + "unsafe" +) + +// Event is the type of the notification messages +// received on the watcher's Event channel. +type Event struct { + Mask uint32 // Mask of events + Cookie uint32 // Unique cookie associating related events (for rename) + Name string // File name (optional) +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + reply chan os.Error +} + +type inode struct { + handle syscall.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov syscall.Overlapped + ino *inode // i-number + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf [4096]byte +} + +type indexMap map[uint64]*watch +type watchMap map[uint32]indexMap + +// A Watcher waits for and receives event notifications +// for a specific set of files and directories. +type Watcher struct { + port syscall.Handle // Handle to completion port + watches watchMap // Map of watches (key: i-number) + input chan *input // Inputs to the reader are sent on this channel + Event chan *Event // Events are returned on this channel + Error chan os.Error // Errors are sent on this channel + isClosed bool // Set to true when Close() is first called + quit chan chan<- os.Error + cookie uint32 +} + +// NewWatcher creates and returns a Watcher. +func NewWatcher() (*Watcher, os.Error) { + port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) + if e != 0 { + return nil, os.NewSyscallError("CreateIoCompletionPort", e) + } + w := &Watcher{ + port: port, + watches: make(watchMap), + input: make(chan *input, 1), + Event: make(chan *Event, 50), + Error: make(chan os.Error), + quit: make(chan chan<- os.Error, 1), + } + go w.readEvents() + return w, nil +} + +// Close closes a Watcher. +// It sends a message to the reader goroutine to quit and removes all watches +// associated with the watcher. +func (w *Watcher) Close() os.Error { + if w.isClosed { + return nil + } + w.isClosed = true + + // Send "quit" message to the reader goroutine + ch := make(chan os.Error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +// AddWatch adds path to the watched file set. +func (w *Watcher) AddWatch(path string, flags uint32) os.Error { + if w.isClosed { + return os.NewError("watcher already closed") + } + in := &input{ + op: opAddWatch, + path: filepath.Clean(path), + flags: flags, + reply: make(chan os.Error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +// Watch adds path to the watched file set, watching all events. +func (w *Watcher) Watch(path string) os.Error { + return w.AddWatch(path, FS_ALL_EVENTS) +} + +// RemoveWatch removes path from the watched file set. +func (w *Watcher) RemoveWatch(path string) os.Error { + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(path), + reply: make(chan os.Error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +func (w *Watcher) wakeupReader() os.Error { + e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if e != 0 { + return os.NewSyscallError("PostQueuedCompletionStatus", e) + } + return nil +} + +func getDir(pathname string) (dir string, err os.Error) { + attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) + if e != 0 { + return "", os.NewSyscallError("GetFileAttributes", e) + } + if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func getIno(path string) (ino *inode, err os.Error) { + h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), + syscall.FILE_LIST_DIRECTORY, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) + if e != 0 { + return nil, os.NewSyscallError("CreateFile", e) + } + var fi syscall.ByHandleFileInformation + if e = syscall.GetFileInformationByHandle(h, &fi); e != 0 { + syscall.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", e) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *Watcher) addWatch(pathname string, flags uint64) os.Error { + dir, err := getDir(pathname) + if err != nil { + return err + } + if flags&FS_ONLYDIR != 0 && pathname != dir { + return nil + } + ino, err := getIno(dir) + if err != nil { + return err + } + watchEntry := w.watches.get(ino) + if watchEntry == nil { + if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != 0 { + syscall.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", e) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + } + w.watches.set(ino, watchEntry) + flags |= provisional + } else { + syscall.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + if err = w.startRead(watchEntry); err != nil { + return err + } + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *Watcher) removeWatch(pathname string) os.Error { + dir, err := getDir(pathname) + if err != nil { + return err + } + ino, err := getIno(dir) + if err != nil { + return err + } + watch := w.watches.get(ino) + if watch == nil { + return fmt.Errorf("can't remove non-existent watch for: %s", pathname) + } + if pathname == dir { + w.sendEvent(watch.path, watch.mask&FS_IGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(watch.path+"/"+name, watch.names[name]&FS_IGNORED) + delete(watch.names, name) + } + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *Watcher) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(watch.path+"/"+name, mask&FS_IGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, watch.mask&FS_IGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *Watcher) startRead(watch *watch) os.Error { + if e := syscall.CancelIo(watch.ino.handle); e != 0 { + w.Error <- os.NewSyscallError("CancelIo", e) + w.deleteWatch(watch) + } + mask := toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= toWindowsFlags(m) + } + if mask == 0 { + if e := syscall.CloseHandle(watch.ino.handle); e != 0 { + w.Error <- os.NewSyscallError("CloseHandle", e) + } + delete(w.watches[watch.ino.volume], watch.ino.index) + return nil + } + e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], + uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) + if e != 0 { + err := os.NewSyscallError("ReadDirectoryChanges", e) + if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + if w.sendEvent(watch.path, watch.mask&FS_DELETE_SELF) { + if watch.mask&FS_ONESHOT != 0 { + watch.mask = 0 + } + } + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Event channel. +// Entry point to the I/O thread. +func (w *Watcher) readEvents() { + var ( + n, key uint32 + ov *syscall.Overlapped + ) + runtime.LockOSThread() + + for { + e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) + watch := (*watch)(unsafe.Pointer(ov)) + + if watch == nil { + select { + case ch := <-w.quit: + for _, index := range w.watches { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + var err os.Error + if e := syscall.CloseHandle(w.port); e != 0 { + err = os.NewSyscallError("CloseHandle", e) + } + close(w.Event) + close(w.Error) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags)) + case opRemoveWatch: + in.reply <- w.removeWatch(in.path) + } + default: + } + continue + } + + switch e { + case syscall.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, watch.mask&FS_DELETE_SELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case syscall.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.Error <- os.NewSyscallError("GetQueuedCompletionPort", e) + continue + case 0: + } + + var offset uint32 + for { + if n == 0 { + w.Event <- &Event{Mask: FS_Q_OVERFLOW} + w.Error <- os.NewError("short read in readEvents()") + break + } + + // Point "raw" to the event in the buffer + raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) + name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) + fullname := watch.path + "/" + name + + var mask uint64 + switch raw.Action { + case syscall.FILE_ACTION_REMOVED: + mask = FS_DELETE_SELF + case syscall.FILE_ACTION_MODIFIED: + mask = FS_MODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = FS_MOVE_SELF + } + } + + sendNameEvent := func() { + if w.sendEvent(fullname, watch.names[name]&mask) { + if watch.names[name]&FS_ONESHOT != 0 { + delete(watch.names, name) + } + } + } + if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { + sendNameEvent() + } + if raw.Action == syscall.FILE_ACTION_REMOVED { + w.sendEvent(fullname, watch.names[name]&FS_IGNORED) + delete(watch.names, name) + } + if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { + if watch.mask&FS_ONESHOT != 0 { + watch.mask = 0 + } + } + if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { + fullname = watch.path + "/" + watch.rename + sendNameEvent() + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + } + + if err := w.startRead(watch); err != nil { + w.Error <- err + } + } +} + +func (w *Watcher) sendEvent(name string, mask uint64) bool { + if mask == 0 { + return false + } + event := &Event{Mask: uint32(mask), Name: name} + if mask&FS_MOVE != 0 { + if mask&FS_MOVED_FROM != 0 { + w.cookie++ + } + event.Cookie = w.cookie + } + select { + case ch := <-w.quit: + w.quit <- ch + case w.Event <- event: + } + return true +} + +// String formats the event e in the form +// "filename: 0xEventMask = FS_ACCESS|FS_ATTRIB_|..." +func (e *Event) String() string { + var events string + m := e.Mask + for _, b := range eventBits { + if m&b.Value != 0 { + m &^= b.Value + events += "|" + b.Name + } + } + if m != 0 { + events += fmt.Sprintf("|%#x", m) + } + if len(events) > 0 { + events = " == " + events[1:] + } + return fmt.Sprintf("%q: %#x%s", e.Name, e.Mask, events) +} + +func toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&FS_ACCESS != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS + } + if mask&FS_MODIFY != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&FS_ATTRIB != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES + } + if mask&(FS_MOVE|FS_CREATE|FS_DELETE) != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func toFSnotifyFlags(action uint32) uint64 { + switch action { + case syscall.FILE_ACTION_ADDED: + return FS_CREATE + case syscall.FILE_ACTION_REMOVED: + return FS_DELETE + case syscall.FILE_ACTION_MODIFIED: + return FS_MODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + return FS_MOVED_FROM + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + return FS_MOVED_TO + } + return 0 +} + +const ( + // Options for AddWatch + FS_ONESHOT = 0x80000000 + FS_ONLYDIR = 0x1000000 + + // Events + FS_ACCESS = 0x1 + FS_ALL_EVENTS = 0xfff + FS_ATTRIB = 0x4 + FS_CLOSE = 0x18 + FS_CREATE = 0x100 + FS_DELETE = 0x200 + FS_DELETE_SELF = 0x400 + FS_MODIFY = 0x2 + FS_MOVE = 0xc0 + FS_MOVED_FROM = 0x40 + FS_MOVED_TO = 0x80 + FS_MOVE_SELF = 0x800 + + // Special events + FS_IGNORED = 0x8000 + FS_Q_OVERFLOW = 0x4000 +) + +var eventBits = []struct { + Value uint32 + Name string +}{ + {FS_ACCESS, "FS_ACCESS"}, + {FS_ATTRIB, "FS_ATTRIB"}, + {FS_CREATE, "FS_CREATE"}, + {FS_DELETE, "FS_DELETE"}, + {FS_DELETE_SELF, "FS_DELETE_SELF"}, + {FS_MODIFY, "FS_MODIFY"}, + {FS_MOVED_FROM, "FS_MOVED_FROM"}, + {FS_MOVED_TO, "FS_MOVED_TO"}, + {FS_MOVE_SELF, "FS_MOVE_SELF"}, + {FS_IGNORED, "FS_IGNORED"}, + {FS_Q_OVERFLOW, "FS_Q_OVERFLOW"}, +} diff --git a/libgo/go/exp/winfsnotify/winfsnotify_test.go b/libgo/go/exp/winfsnotify/winfsnotify_test.go new file mode 100644 index 0000000..edf2165 --- /dev/null +++ b/libgo/go/exp/winfsnotify/winfsnotify_test.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package winfsnotify + +import ( + "os" + "time" + "testing" +) + +func expect(t *testing.T, eventstream <-chan *Event, name string, mask uint32) { + t.Logf(`expected: "%s": 0x%x`, name, mask) + select { + case event := <-eventstream: + if event == nil { + t.Fatal("nil event received") + } + t.Logf("received: %s", event) + if event.Name != name || event.Mask != mask { + t.Fatal("did not receive expected event") + } + case <-time.After(1e9): + t.Fatal("timed out waiting for event") + } +} + +func TestNotifyEvents(t *testing.T) { + watcher, err := NewWatcher() + if err != nil { + t.Fatalf("NewWatcher() failed: %s", err) + } + + testDir := "TestNotifyEvents.testdirectory" + testFile := testDir + "/TestNotifyEvents.testfile" + testFile2 := testFile + ".new" + const mask = FS_ALL_EVENTS & ^(FS_ATTRIB|FS_CLOSE) | FS_IGNORED + + // Add a watch for testDir + os.RemoveAll(testDir) + if err = os.Mkdir(testDir, 0777); err != nil { + t.Fatalf("Failed to create test directory", err) + } + defer os.RemoveAll(testDir) + err = watcher.AddWatch(testDir, mask) + if err != nil { + t.Fatalf("Watcher.Watch() failed: %s", err) + } + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + // Create a file + file, err := os.Create(testFile) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + expect(t, watcher.Event, testFile, FS_CREATE) + + err = watcher.AddWatch(testFile, mask) + if err != nil { + t.Fatalf("Watcher.Watch() failed: %s", err) + } + + if _, err = file.WriteString("hello, world"); err != nil { + t.Fatalf("failed to write to test file: %s", err) + } + if err = file.Sync(); err != nil { + t.Fatalf("failed to sync test file: %s", err) + } + expect(t, watcher.Event, testFile, FS_MODIFY) + expect(t, watcher.Event, testFile, FS_MODIFY) + + if err = file.Close(); err != nil { + t.Fatalf("failed to close test file: %s", err) + } + + if err = os.Rename(testFile, testFile2); err != nil { + t.Fatalf("failed to rename test file: %s", err) + } + expect(t, watcher.Event, testFile, FS_MOVED_FROM) + expect(t, watcher.Event, testFile2, FS_MOVED_TO) + expect(t, watcher.Event, testFile, FS_MOVE_SELF) + + if err = os.RemoveAll(testDir); err != nil { + t.Fatalf("failed to remove test directory: %s", err) + } + expect(t, watcher.Event, testFile2, FS_DELETE_SELF) + expect(t, watcher.Event, testFile2, FS_IGNORED) + expect(t, watcher.Event, testFile2, FS_DELETE) + expect(t, watcher.Event, testDir, FS_DELETE_SELF) + expect(t, watcher.Event, testDir, FS_IGNORED) + + t.Log("calling Close()") + if err = watcher.Close(); err != nil { + t.Fatalf("failed to close watcher: %s", err) + } +} + +func TestNotifyClose(t *testing.T) { + watcher, _ := NewWatcher() + watcher.Close() + + done := false + go func() { + watcher.Close() + done = true + }() + + time.Sleep(50e6) // 50 ms + if !done { + t.Fatal("double Close() test failed: second Close() call didn't return") + } + + err := watcher.Watch("_test") + if err == nil { + t.Fatal("expected error on Watch() after Close(), got nil") + } +} diff --git a/libgo/go/exp/wingui/gui.go b/libgo/go/exp/wingui/gui.go index cf39293..a2f16f2 100644 --- a/libgo/go/exp/wingui/gui.go +++ b/libgo/go/exp/wingui/gui.go @@ -25,13 +25,12 @@ func abortErrNo(funcname string, err int) { // global vars var ( - mh uint32 - bh uint32 + mh syscall.Handle + bh syscall.Handle ) // WinProc called by windows to notify us of all windows events we might be interested in. -func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr { - var rc int32 +func WndProc(hwnd syscall.Handle, msg uint32, wparam, lparam uintptr) (rc uintptr) { switch msg { case WM_CREATE: var e int @@ -49,7 +48,7 @@ func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr { fmt.Printf("button handle is %x\n", bh) rc = DefWindowProc(hwnd, msg, wparam, lparam) case WM_COMMAND: - switch uint32(lparam) { + switch syscall.Handle(lparam) { case bh: e := PostMessage(hwnd, WM_CLOSE, 0, 0) if e != 0 { @@ -66,7 +65,7 @@ func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr { rc = DefWindowProc(hwnd, msg, wparam, lparam) } //fmt.Printf("WndProc(0x%08x, %d, 0x%08x, 0x%08x) (%d)\n", hwnd, msg, wparam, lparam, rc) - return uintptr(rc) + return } func rungui() int { diff --git a/libgo/go/exp/wingui/winapi.go b/libgo/go/exp/wingui/winapi.go index 31b57a2..3201528 100644 --- a/libgo/go/exp/wingui/winapi.go +++ b/libgo/go/exp/wingui/winapi.go @@ -6,6 +6,7 @@ package main import ( "unsafe" + "syscall" ) type Wndclassex struct { @@ -14,25 +15,25 @@ type Wndclassex struct { WndProc uintptr ClsExtra int32 WndExtra int32 - Instance uint32 - Icon uint32 - Cursor uint32 - Background uint32 + Instance syscall.Handle + Icon syscall.Handle + Cursor syscall.Handle + Background syscall.Handle MenuName *uint16 ClassName *uint16 - IconSm uint32 + IconSm syscall.Handle } type Point struct { - X int32 - Y int32 + X uintptr + Y uintptr } type Msg struct { - Hwnd uint32 + Hwnd syscall.Handle Message uint32 - Wparam int32 - Lparam int32 + Wparam uintptr + Lparam uintptr Time uint32 Pt Point } @@ -109,22 +110,22 @@ var ( IDI_INFORMATION = IDI_ASTERISK ) -//sys GetModuleHandle(modname *uint16) (handle uint32, errno int) = GetModuleHandleW +//sys GetModuleHandle(modname *uint16) (handle syscall.Handle, errno int) = GetModuleHandleW //sys RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) = user32.RegisterClassExW -//sys CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) = user32.CreateWindowExW -//sys DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.DefWindowProcW -//sys DestroyWindow(hwnd uint32) (errno int) = user32.DestroyWindow +//sys CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent syscall.Handle, menu syscall.Handle, instance syscall.Handle, param uintptr) (hwnd syscall.Handle, errno int) = user32.CreateWindowExW +//sys DefWindowProc(hwnd syscall.Handle, msg uint32, wparam uintptr, lparam uintptr) (lresult uintptr) = user32.DefWindowProcW +//sys DestroyWindow(hwnd syscall.Handle) (errno int) = user32.DestroyWindow //sys PostQuitMessage(exitcode int32) = user32.PostQuitMessage -//sys ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) = user32.ShowWindow -//sys UpdateWindow(hwnd uint32) (errno int) = user32.UpdateWindow -//sys GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) [failretval==-1] = user32.GetMessageW +//sys ShowWindow(hwnd syscall.Handle, cmdshow int32) (wasvisible bool) = user32.ShowWindow +//sys UpdateWindow(hwnd syscall.Handle) (errno int) = user32.UpdateWindow +//sys GetMessage(msg *Msg, hwnd syscall.Handle, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) [failretval==-1] = user32.GetMessageW //sys TranslateMessage(msg *Msg) (done bool) = user32.TranslateMessage //sys DispatchMessage(msg *Msg) (ret int32) = user32.DispatchMessageW -//sys LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) = user32.LoadIconW -//sys LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) = user32.LoadCursorW -//sys SetCursor(cursor uint32) (precursor uint32, errno int) = user32.SetCursor -//sys SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.SendMessageW -//sys PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) = user32.PostMessageW +//sys LoadIcon(instance syscall.Handle, iconname *uint16) (icon syscall.Handle, errno int) = user32.LoadIconW +//sys LoadCursor(instance syscall.Handle, cursorname *uint16) (cursor syscall.Handle, errno int) = user32.LoadCursorW +//sys SetCursor(cursor syscall.Handle) (precursor syscall.Handle, errno int) = user32.SetCursor +//sys SendMessage(hwnd syscall.Handle, msg uint32, wparam uintptr, lparam uintptr) (lresult uintptr) = user32.SendMessageW +//sys PostMessage(hwnd syscall.Handle, msg uint32, wparam uintptr, lparam uintptr) (errno int) = user32.PostMessageW func MakeIntResource(id uint16) *uint16 { return (*uint16)(unsafe.Pointer(uintptr(id))) diff --git a/libgo/go/exp/wingui/zwinapi.go b/libgo/go/exp/wingui/zwinapi.go index 4c009dd..38e93ee 100644 --- a/libgo/go/exp/wingui/zwinapi.go +++ b/libgo/go/exp/wingui/zwinapi.go @@ -28,9 +28,9 @@ var ( procPostMessageW = moduser32.NewProc("PostMessageW") ) -func GetModuleHandle(modname *uint16) (handle uint32, errno int) { +func GetModuleHandle(modname *uint16) (handle syscall.Handle, errno int) { r0, _, e1 := syscall.Syscall(procGetModuleHandleW.Addr(), 1, uintptr(unsafe.Pointer(modname)), 0, 0) - handle = uint32(r0) + handle = syscall.Handle(r0) if handle == 0 { if e1 != 0 { errno = int(e1) @@ -58,9 +58,9 @@ func RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) { return } -func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) { +func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent syscall.Handle, menu syscall.Handle, instance syscall.Handle, param uintptr) (hwnd syscall.Handle, errno int) { r0, _, e1 := syscall.Syscall12(procCreateWindowExW.Addr(), 12, uintptr(exstyle), uintptr(unsafe.Pointer(classname)), uintptr(unsafe.Pointer(windowname)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height), uintptr(wndparent), uintptr(menu), uintptr(instance), uintptr(param)) - hwnd = uint32(r0) + hwnd = syscall.Handle(r0) if hwnd == 0 { if e1 != 0 { errno = int(e1) @@ -73,13 +73,13 @@ func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style return } -func DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { +func DefWindowProc(hwnd syscall.Handle, msg uint32, wparam uintptr, lparam uintptr) (lresult uintptr) { r0, _, _ := syscall.Syscall6(procDefWindowProcW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) - lresult = int32(r0) + lresult = uintptr(r0) return } -func DestroyWindow(hwnd uint32) (errno int) { +func DestroyWindow(hwnd syscall.Handle) (errno int) { r1, _, e1 := syscall.Syscall(procDestroyWindow.Addr(), 1, uintptr(hwnd), 0, 0) if int(r1) == 0 { if e1 != 0 { @@ -98,13 +98,13 @@ func PostQuitMessage(exitcode int32) { return } -func ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) { +func ShowWindow(hwnd syscall.Handle, cmdshow int32) (wasvisible bool) { r0, _, _ := syscall.Syscall(procShowWindow.Addr(), 2, uintptr(hwnd), uintptr(cmdshow), 0) wasvisible = bool(r0 != 0) return } -func UpdateWindow(hwnd uint32) (errno int) { +func UpdateWindow(hwnd syscall.Handle) (errno int) { r1, _, e1 := syscall.Syscall(procUpdateWindow.Addr(), 1, uintptr(hwnd), 0, 0) if int(r1) == 0 { if e1 != 0 { @@ -118,7 +118,7 @@ func UpdateWindow(hwnd uint32) (errno int) { return } -func GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) { +func GetMessage(msg *Msg, hwnd syscall.Handle, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) { r0, _, e1 := syscall.Syscall6(procGetMessageW.Addr(), 4, uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(MsgFilterMin), uintptr(MsgFilterMax), 0, 0) ret = int32(r0) if ret == -1 { @@ -145,9 +145,9 @@ func DispatchMessage(msg *Msg) (ret int32) { return } -func LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) { +func LoadIcon(instance syscall.Handle, iconname *uint16) (icon syscall.Handle, errno int) { r0, _, e1 := syscall.Syscall(procLoadIconW.Addr(), 2, uintptr(instance), uintptr(unsafe.Pointer(iconname)), 0) - icon = uint32(r0) + icon = syscall.Handle(r0) if icon == 0 { if e1 != 0 { errno = int(e1) @@ -160,9 +160,9 @@ func LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) { return } -func LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) { +func LoadCursor(instance syscall.Handle, cursorname *uint16) (cursor syscall.Handle, errno int) { r0, _, e1 := syscall.Syscall(procLoadCursorW.Addr(), 2, uintptr(instance), uintptr(unsafe.Pointer(cursorname)), 0) - cursor = uint32(r0) + cursor = syscall.Handle(r0) if cursor == 0 { if e1 != 0 { errno = int(e1) @@ -175,9 +175,9 @@ func LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) return } -func SetCursor(cursor uint32) (precursor uint32, errno int) { +func SetCursor(cursor syscall.Handle) (precursor syscall.Handle, errno int) { r0, _, e1 := syscall.Syscall(procSetCursor.Addr(), 1, uintptr(cursor), 0, 0) - precursor = uint32(r0) + precursor = syscall.Handle(r0) if precursor == 0 { if e1 != 0 { errno = int(e1) @@ -190,13 +190,13 @@ func SetCursor(cursor uint32) (precursor uint32, errno int) { return } -func SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { +func SendMessage(hwnd syscall.Handle, msg uint32, wparam uintptr, lparam uintptr) (lresult uintptr) { r0, _, _ := syscall.Syscall6(procSendMessageW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) - lresult = int32(r0) + lresult = uintptr(r0) return } -func PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) { +func PostMessage(hwnd syscall.Handle, msg uint32, wparam uintptr, lparam uintptr) (errno int) { r1, _, e1 := syscall.Syscall6(procPostMessageW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) if int(r1) == 0 { if e1 != 0 { diff --git a/libgo/go/flag/flag.go b/libgo/go/flag/flag.go index 01bbc37..f13f7a4 100644 --- a/libgo/go/flag/flag.go +++ b/libgo/go/flag/flag.go @@ -204,6 +204,7 @@ type FlagSet struct { Usage func() name string + parsed bool actual map[string]*Flag formal map[string]*Flag args []string // arguments after flags @@ -285,6 +286,9 @@ func (f *FlagSet) Set(name, value string) bool { if !ok { return false } + if f.actual == nil { + f.actual = make(map[string]*Flag) + } f.actual[name] = flag return true } @@ -318,10 +322,15 @@ func defaultUsage(f *FlagSet) { f.PrintDefaults() } +// NOTE: Usage is not just defaultUsage(commandLine) +// because it serves (via godoc flag Usage) as the example +// for how to write your own usage function. + // Usage prints to standard error a usage message documenting all defined command-line flags. // The function is a variable that may be changed to point to a custom function. var Usage = func() { - defaultUsage(commandLine) + fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + PrintDefaults() } // NFlag returns the number of flags that have been set. @@ -553,6 +562,9 @@ func (f *FlagSet) Var(value Value, name string, usage string) { fmt.Fprintf(os.Stderr, "%s flag redefined: %s\n", f.name, name) panic("flag redefinition") // Happens only if flags are declared with identical names } + if f.formal == nil { + f.formal = make(map[string]*Flag) + } f.formal[name] = flag } @@ -580,6 +592,8 @@ func (f *FlagSet) failf(format string, a ...interface{}) os.Error { func (f *FlagSet) usage() { if f == commandLine { Usage() + } else if f.Usage == nil { + defaultUsage(f) } else { f.Usage() } @@ -651,6 +665,9 @@ func (f *FlagSet) parseOne() (bool, os.Error) { return false, f.failf("invalid value %q for flag: -%s", value, name) } } + if f.actual == nil { + f.actual = make(map[string]*Flag) + } f.actual[name] = flag return true, nil } @@ -660,6 +677,7 @@ func (f *FlagSet) parseOne() (bool, os.Error) { // are defined and before flags are accessed by the program. // The return value will be ErrHelp if -help was set but not defined. func (f *FlagSet) Parse(arguments []string) os.Error { + f.parsed = true f.args = arguments for { seen, err := f.parseOne() @@ -681,6 +699,11 @@ func (f *FlagSet) Parse(arguments []string) os.Error { return nil } +// Parsed reports whether f.Parse has been called. +func (f *FlagSet) Parsed() bool { + return f.parsed +} + // Parse parses the command-line flags from os.Args[1:]. Must be called // after all flags are defined and before flags are accessed by the program. func Parse() { @@ -688,6 +711,11 @@ func Parse() { commandLine.Parse(os.Args[1:]) } +// Parsed returns true if the command-line flags have been parsed. +func Parsed() bool { + return commandLine.Parsed() +} + // The default set of command-line flags, parsed from os.Args. var commandLine = NewFlagSet(os.Args[0], ExitOnError) @@ -696,10 +724,15 @@ var commandLine = NewFlagSet(os.Args[0], ExitOnError) func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet { f := &FlagSet{ name: name, - actual: make(map[string]*Flag), - formal: make(map[string]*Flag), errorHandling: errorHandling, } - f.Usage = func() { defaultUsage(f) } return f } + +// Init sets the name and error handling property for a flag set. +// By default, the zero FlagSet uses an empty name and the +// ContinueOnError error handling policy. +func (f *FlagSet) Init(name string, errorHandling ErrorHandling) { + f.name = name + f.errorHandling = errorHandling +} diff --git a/libgo/go/flag/flag_test.go b/libgo/go/flag/flag_test.go index 63d0a9f..f135316 100644 --- a/libgo/go/flag/flag_test.go +++ b/libgo/go/flag/flag_test.go @@ -98,6 +98,9 @@ func TestUsage(t *testing.T) { } func testParse(f *FlagSet, t *testing.T) { + if f.Parsed() { + t.Error("f.Parse() = true before Parse") + } boolFlag := f.Bool("bool", false, "bool value") bool2Flag := f.Bool("bool2", false, "bool2 value") intFlag := f.Int("int", 0, "int value") @@ -121,6 +124,9 @@ func testParse(f *FlagSet, t *testing.T) { if err := f.Parse(args); err != nil { t.Fatal(err) } + if !f.Parsed() { + t.Error("f.Parse() = false after Parse") + } if *boolFlag != true { t.Error("bool flag should be true, is ", *boolFlag) } @@ -174,7 +180,8 @@ func (f *flagVar) Set(value string) bool { } func TestUserDefined(t *testing.T) { - flags := NewFlagSet("test", ContinueOnError) + var flags FlagSet + flags.Init("test", ContinueOnError) var v flagVar flags.Var(&v, "v", "usage") if err := flags.Parse([]string{"-v", "1", "-v", "2", "-v=3"}); err != nil { diff --git a/libgo/go/fmt/doc.go b/libgo/go/fmt/doc.go index 35a11e1..c993e57 100644 --- a/libgo/go/fmt/doc.go +++ b/libgo/go/fmt/doc.go @@ -122,7 +122,7 @@ An analogous set of functions scans formatted text to yield values. Scan, Scanf and Scanln read from os.Stdin; Fscan, - Fscanf and Fscanln read from a specified os.Reader; Sscan, + Fscanf and Fscanln read from a specified io.Reader; Sscan, Sscanf and Sscanln read from an argument string. Scanln, Fscanln and Sscanln stop scanning at a newline and require that the items be followed by one; Sscanf, Fscanf and Sscanf require diff --git a/libgo/go/fmt/fmt_test.go b/libgo/go/fmt/fmt_test.go index 38218df..030ad61 100644 --- a/libgo/go/fmt/fmt_test.go +++ b/libgo/go/fmt/fmt_test.go @@ -5,6 +5,7 @@ package fmt_test import ( + "bytes" . "fmt" "io" "math" @@ -348,8 +349,8 @@ var fmttests = []struct { {"%#v", make(chan int), "(chan int)(0xPTR)"}, {"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"}, {"%#v", 1000000000, "1000000000"}, - {"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`}, - {"%#v", map[string]B{"a": {1, 2}, "b": {3, 4}}, `map[string] fmt_test.B{"a":fmt_test.B{I:1, j:2}, "b":fmt_test.B{I:3, j:4}}`}, + {"%#v", map[string]int{"a": 1}, `map[string] int{"a":1}`}, + {"%#v", map[string]B{"a": {1, 2}}, `map[string] fmt_test.B{"a":fmt_test.B{I:1, j:2}}`}, {"%#v", []string{"a", "b"}, `[]string{"a", "b"}`}, // slices with other formats @@ -504,11 +505,38 @@ func TestCountMallocs(t *testing.T) { runtime.UpdateMemStats() mallocs = 0 - runtime.MemStats.Mallocs for i := 0; i < 100; i++ { + Sprintf("%s", "hello") + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%s\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { Sprintf("%x %x", i, i) } runtime.UpdateMemStats() mallocs += runtime.MemStats.Mallocs Printf("mallocs per Sprintf(\"%%x %%x\"): %d\n", mallocs/100) + buf := new(bytes.Buffer) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + buf.Reset() + Fprintf(buf, "%x %x %x", i, i, i) + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Fprintf(buf, \"%%x %%x %%x\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + buf.Reset() + Fprintf(buf, "%s", "hello") + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Fprintf(buf, \"%%s\"): %d\n", mallocs/100) } type flagPrinter struct{} diff --git a/libgo/go/fmt/print.go b/libgo/go/fmt/print.go index 136aebd..7721e72 100644 --- a/libgo/go/fmt/print.go +++ b/libgo/go/fmt/print.go @@ -9,6 +9,7 @@ import ( "io" "os" "reflect" + "sync" "unicode" "utf8" ) @@ -73,39 +74,45 @@ type pp struct { n int panicking bool buf bytes.Buffer - runeBuf [utf8.UTFMax]byte - fmt fmt + // value holds the current item, as a reflect.Value, and will be + // the zero Value if the item has not been reflected. + value reflect.Value + runeBuf [utf8.UTFMax]byte + fmt fmt } // A cache holds a set of reusable objects. -// The buffered channel holds the currently available objects. +// The slice is a stack (LIFO). // If more are needed, the cache creates them by calling new. type cache struct { - saved chan interface{} + mu sync.Mutex + saved []interface{} new func() interface{} } func (c *cache) put(x interface{}) { - select { - case c.saved <- x: - // saved in cache - default: - // discard + c.mu.Lock() + if len(c.saved) < cap(c.saved) { + c.saved = append(c.saved, x) } + c.mu.Unlock() } func (c *cache) get() interface{} { - select { - case x := <-c.saved: - return x // reused from cache - default: + c.mu.Lock() + n := len(c.saved) + if n == 0 { + c.mu.Unlock() return c.new() } - panic("not reached") + x := c.saved[n-1] + c.saved = c.saved[0 : n-1] + c.mu.Unlock() + return x } func newCache(f func() interface{}) *cache { - return &cache{make(chan interface{}, 100), f} + return &cache{saved: make([]interface{}, 0, 100), new: f} } var ppFree = newCache(func() interface{} { return new(pp) }) @@ -125,6 +132,7 @@ func (p *pp) free() { return } p.buf.Reset() + p.value = reflect.Value{} ppFree.put(p) } @@ -186,7 +194,7 @@ func Sprintf(format string, a ...interface{}) string { } // Errorf formats according to a format specifier and returns the string -// converted to an os.ErrorString, which satisfies the os.Error interface. +// as a value that satisfies os.Error. func Errorf(format string, a ...interface{}) os.Error { return os.NewError(Sprintf(format, a...)) } @@ -286,7 +294,7 @@ func (p *pp) unknownType(v interface{}) { p.buf.WriteByte('?') } -func (p *pp) badVerb(verb int, val interface{}, val1 reflect.Value) { +func (p *pp) badVerb(verb int, val interface{}) { p.add('%') p.add('!') p.add(verb) @@ -296,22 +304,22 @@ func (p *pp) badVerb(verb int, val interface{}, val1 reflect.Value) { p.buf.WriteString(reflect.TypeOf(val).String()) p.add('=') p.printField(val, 'v', false, false, 0) - case val1.IsValid(): - p.buf.WriteString(val1.Type().String()) + case p.value.IsValid(): + p.buf.WriteString(p.value.Type().String()) p.add('=') - p.printValue(val1, 'v', false, false, 0) + p.printValue(p.value, 'v', false, false, 0) default: p.buf.Write(nilAngleBytes) } p.add(')') } -func (p *pp) fmtBool(v bool, verb int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtBool(v bool, verb int, value interface{}) { switch verb { case 't', 'v': p.fmt.fmt_boolean(v) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } @@ -325,7 +333,7 @@ func (p *pp) fmtC(c int64) { p.fmt.pad(p.runeBuf[0:w]) } -func (p *pp) fmtInt64(v int64, verb int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtInt64(v int64, verb int, value interface{}) { switch verb { case 'b': p.fmt.integer(v, 2, signed, ldigits) @@ -339,7 +347,7 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}, value1 reflect.Value if 0 <= v && v <= unicode.MaxRune { p.fmt.fmt_qc(v) } else { - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } case 'x': p.fmt.integer(v, 16, signed, ldigits) @@ -348,7 +356,7 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}, value1 reflect.Value case 'X': p.fmt.integer(v, 16, signed, udigits) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } @@ -383,7 +391,7 @@ func (p *pp) fmtUnicode(v int64) { p.fmt.sharp = sharp } -func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}, value1 reflect.Value) { +func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { switch verb { case 'b': p.fmt.integer(int64(v), 2, unsigned, ldigits) @@ -403,7 +411,7 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}, val if 0 <= v && v <= unicode.MaxRune { p.fmt.fmt_qc(int64(v)) } else { - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } case 'x': p.fmt.integer(int64(v), 16, unsigned, ldigits) @@ -412,11 +420,11 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}, val case 'U': p.fmtUnicode(int64(v)) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } -func (p *pp) fmtFloat32(v float32, verb int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtFloat32(v float32, verb int, value interface{}) { switch verb { case 'b': p.fmt.fmt_fb32(v) @@ -431,11 +439,11 @@ func (p *pp) fmtFloat32(v float32, verb int, value interface{}, value1 reflect.V case 'G': p.fmt.fmt_G32(v) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } -func (p *pp) fmtFloat64(v float64, verb int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtFloat64(v float64, verb int, value interface{}) { switch verb { case 'b': p.fmt.fmt_fb64(v) @@ -450,33 +458,33 @@ func (p *pp) fmtFloat64(v float64, verb int, value interface{}, value1 reflect.V case 'G': p.fmt.fmt_G64(v) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } -func (p *pp) fmtComplex64(v complex64, verb int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtComplex64(v complex64, verb int, value interface{}) { switch verb { case 'e', 'E', 'f', 'F', 'g', 'G': p.fmt.fmt_c64(v, verb) case 'v': p.fmt.fmt_c64(v, 'g') default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } -func (p *pp) fmtComplex128(v complex128, verb int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtComplex128(v complex128, verb int, value interface{}) { switch verb { case 'e', 'E', 'f', 'F', 'g', 'G': p.fmt.fmt_c128(v, verb) case 'v': p.fmt.fmt_c128(v, 'g') default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } -func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}, value1 reflect.Value) { +func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}) { switch verb { case 'v': if goSyntax { @@ -493,11 +501,11 @@ func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}, val case 'q': p.fmt.fmt_q(v) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } -func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interface{}, value1 reflect.Value) { +func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interface{}) { if verb == 'v' || verb == 'd' { if goSyntax { p.buf.Write(bytesBytes) @@ -532,7 +540,7 @@ func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interf case 'q': p.fmt.fmt_q(s) default: - p.badVerb(verb, value, value1) + p.badVerb(verb, value) } } @@ -542,7 +550,7 @@ func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSynt case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: u = value.Pointer() default: - p.badVerb(verb, field, value) + p.badVerb(verb, field) return } if goSyntax { @@ -615,7 +623,7 @@ func (p *pp) handleMethods(field interface{}, verb int, plus, goSyntax bool, dep handled = true defer p.catchPanic(field, verb) // Print the result of GoString unadorned. - p.fmtString(stringer.GoString(), 's', false, field, reflect.Value{}) + p.fmtString(stringer.GoString(), 's', false, field) return } } else { @@ -637,7 +645,7 @@ func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth if verb == 'T' || verb == 'v' { p.buf.Write(nilAngleBytes) } else { - p.badVerb(verb, field, reflect.Value{}) + p.badVerb(verb, field) } return false } @@ -660,58 +668,58 @@ func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth // Some types can be done without reflection. switch f := field.(type) { case bool: - p.fmtBool(f, verb, field, reflect.Value{}) + p.fmtBool(f, verb, field) return false case float32: - p.fmtFloat32(f, verb, field, reflect.Value{}) + p.fmtFloat32(f, verb, field) return false case float64: - p.fmtFloat64(f, verb, field, reflect.Value{}) + p.fmtFloat64(f, verb, field) return false case complex64: - p.fmtComplex64(complex64(f), verb, field, reflect.Value{}) + p.fmtComplex64(complex64(f), verb, field) return false case complex128: - p.fmtComplex128(f, verb, field, reflect.Value{}) + p.fmtComplex128(f, verb, field) return false case int: - p.fmtInt64(int64(f), verb, field, reflect.Value{}) + p.fmtInt64(int64(f), verb, field) return false case int8: - p.fmtInt64(int64(f), verb, field, reflect.Value{}) + p.fmtInt64(int64(f), verb, field) return false case int16: - p.fmtInt64(int64(f), verb, field, reflect.Value{}) + p.fmtInt64(int64(f), verb, field) return false case int32: - p.fmtInt64(int64(f), verb, field, reflect.Value{}) + p.fmtInt64(int64(f), verb, field) return false case int64: - p.fmtInt64(f, verb, field, reflect.Value{}) + p.fmtInt64(f, verb, field) return false case uint: - p.fmtUint64(uint64(f), verb, goSyntax, field, reflect.Value{}) + p.fmtUint64(uint64(f), verb, goSyntax, field) return false case uint8: - p.fmtUint64(uint64(f), verb, goSyntax, field, reflect.Value{}) + p.fmtUint64(uint64(f), verb, goSyntax, field) return false case uint16: - p.fmtUint64(uint64(f), verb, goSyntax, field, reflect.Value{}) + p.fmtUint64(uint64(f), verb, goSyntax, field) return false case uint32: - p.fmtUint64(uint64(f), verb, goSyntax, field, reflect.Value{}) + p.fmtUint64(uint64(f), verb, goSyntax, field) return false case uint64: - p.fmtUint64(f, verb, goSyntax, field, reflect.Value{}) + p.fmtUint64(f, verb, goSyntax, field) return false case uintptr: - p.fmtUint64(uint64(f), verb, goSyntax, field, reflect.Value{}) + p.fmtUint64(uint64(f), verb, goSyntax, field) return false case string: - p.fmtString(f, verb, goSyntax, field, reflect.Value{}) + p.fmtString(f, verb, goSyntax, field) return verb == 's' || verb == 'v' case []byte: - p.fmtBytes(f, verb, goSyntax, depth, field, reflect.Value{}) + p.fmtBytes(f, verb, goSyntax, depth, field) return verb == 's' } @@ -725,7 +733,7 @@ func (p *pp) printValue(value reflect.Value, verb int, plus, goSyntax bool, dept if verb == 'T' || verb == 'v' { p.buf.Write(nilAngleBytes) } else { - p.badVerb(verb, nil, value) + p.badVerb(verb, nil) } return false } @@ -757,28 +765,30 @@ func (p *pp) printValue(value reflect.Value, verb int, plus, goSyntax bool, dept // printReflectValue is the fallback for both printField and printValue. // It uses reflect to print the value. func (p *pp) printReflectValue(value reflect.Value, verb int, plus, goSyntax bool, depth int) (wasString bool) { + oldValue := p.value + p.value = value BigSwitch: switch f := value; f.Kind() { case reflect.Bool: - p.fmtBool(f.Bool(), verb, nil, value) + p.fmtBool(f.Bool(), verb, nil) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - p.fmtInt64(f.Int(), verb, nil, value) + p.fmtInt64(f.Int(), verb, nil) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - p.fmtUint64(uint64(f.Uint()), verb, goSyntax, nil, value) + p.fmtUint64(uint64(f.Uint()), verb, goSyntax, nil) case reflect.Float32, reflect.Float64: if f.Type().Size() == 4 { - p.fmtFloat32(float32(f.Float()), verb, nil, value) + p.fmtFloat32(float32(f.Float()), verb, nil) } else { - p.fmtFloat64(float64(f.Float()), verb, nil, value) + p.fmtFloat64(float64(f.Float()), verb, nil) } case reflect.Complex64, reflect.Complex128: if f.Type().Size() == 8 { - p.fmtComplex64(complex64(f.Complex()), verb, nil, value) + p.fmtComplex64(complex64(f.Complex()), verb, nil) } else { - p.fmtComplex128(complex128(f.Complex()), verb, nil, value) + p.fmtComplex128(complex128(f.Complex()), verb, nil) } case reflect.String: - p.fmtString(f.String(), verb, goSyntax, nil, value) + p.fmtString(f.String(), verb, goSyntax, nil) case reflect.Map: if goSyntax { p.buf.WriteString(f.Type().String()) @@ -838,7 +848,7 @@ BigSwitch: p.buf.Write(nilAngleBytes) } } else { - return p.printValue(value, verb, plus, goSyntax, depth+1) + wasString = p.printValue(value, verb, plus, goSyntax, depth+1) } case reflect.Array, reflect.Slice: // Byte slices are special. @@ -854,8 +864,9 @@ BigSwitch: for i := range bytes { bytes[i] = byte(f.Index(i).Uint()) } - p.fmtBytes(bytes, verb, goSyntax, depth, nil, value) - return verb == 's' + p.fmtBytes(bytes, verb, goSyntax, depth, nil) + wasString = verb == 's' + break } if goSyntax { p.buf.WriteString(value.Type().String()) @@ -917,7 +928,8 @@ BigSwitch: default: p.unknownType(f) } - return false + p.value = oldValue + return wasString } // intFromArg gets the fieldnumth element of a. On return, isInt reports whether the argument has type int. diff --git a/libgo/go/go/ast/filter.go b/libgo/go/go/ast/filter.go index 2673343..d7d4b4b 100644 --- a/libgo/go/go/ast/filter.go +++ b/libgo/go/go/ast/filter.go @@ -9,10 +9,44 @@ import "go/token" // ---------------------------------------------------------------------------- // Export filtering -func identListExports(list []*Ident) []*Ident { +// exportFilter is a special filter function to extract exported nodes. +func exportFilter(name string) bool { + return IsExported(name) +} + +// FileExports trims the AST for a Go source file in place such that +// only exported nodes remain: all top-level identifiers which are not exported +// and their associated information (such as type, initial value, or function +// body) are removed. Non-exported fields and methods of exported types are +// stripped. The File.Comments list is not changed. +// +// FileExports returns true if there are exported declarationa; +// it returns false otherwise. +// +func FileExports(src *File) bool { + return FilterFile(src, exportFilter) +} + +// PackageExports trims the AST for a Go package in place such that +// only exported nodes remain. The pkg.Files list is not changed, so that +// file names and top-level package comments don't get lost. +// +// PackageExports returns true if there are exported declarations; +// it returns false otherwise. +// +func PackageExports(pkg *Package) bool { + return FilterPackage(pkg, exportFilter) +} + +// ---------------------------------------------------------------------------- +// General filtering + +type Filter func(string) bool + +func filterIdentList(list []*Ident, f Filter) []*Ident { j := 0 for _, x := range list { - if x.IsExported() { + if f(x.Name) { list[j] = x j++ } @@ -38,33 +72,30 @@ func fieldName(x Expr) *Ident { return nil } -func fieldListExports(fields *FieldList) (removedFields bool) { +func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { if fields == nil { - return + return false } list := fields.List j := 0 for _, f := range list { - exported := false + keepField := false if len(f.Names) == 0 { // anonymous field - // (Note that a non-exported anonymous field - // may still refer to a type with exported - // fields, so this is not absolutely correct. - // However, this cannot be done w/o complete - // type information.) name := fieldName(f.Type) - exported = name != nil && name.IsExported() + keepField = name != nil && filter(name.Name) } else { n := len(f.Names) - f.Names = identListExports(f.Names) + f.Names = filterIdentList(f.Names, filter) if len(f.Names) < n { removedFields = true } - exported = len(f.Names) > 0 + keepField = len(f.Names) > 0 } - if exported { - typeExports(f.Type) + if keepField { + if filter == exportFilter { + filterType(f.Type, filter) + } list[j] = f j++ } @@ -76,185 +107,75 @@ func fieldListExports(fields *FieldList) (removedFields bool) { return } -func paramListExports(fields *FieldList) { +func filterParamList(fields *FieldList, filter Filter) bool { if fields == nil { - return + return false } + var b bool for _, f := range fields.List { - typeExports(f.Type) + if filterType(f.Type, filter) { + b = true + } } + return b } -func typeExports(typ Expr) { +func filterType(typ Expr, f Filter) bool { switch t := typ.(type) { + case *Ident: + return f(t.Name) + case *ParenExpr: + return filterType(t.X, f) case *ArrayType: - typeExports(t.Elt) + return filterType(t.Elt, f) case *StructType: - if fieldListExports(t.Fields) { + if filterFieldList(t.Fields, f) { t.Incomplete = true } + return len(t.Fields.List) > 0 case *FuncType: - paramListExports(t.Params) - paramListExports(t.Results) + b1 := filterParamList(t.Params, f) + b2 := filterParamList(t.Results, f) + return b1 || b2 case *InterfaceType: - if fieldListExports(t.Methods) { + if filterFieldList(t.Methods, f) { t.Incomplete = true } + return len(t.Methods.List) > 0 case *MapType: - typeExports(t.Key) - typeExports(t.Value) + b1 := filterType(t.Key, f) + b2 := filterType(t.Value, f) + return b1 || b2 case *ChanType: - typeExports(t.Value) - } -} - -func specExports(spec Spec) bool { - switch s := spec.(type) { - case *ValueSpec: - s.Names = identListExports(s.Names) - if len(s.Names) > 0 { - typeExports(s.Type) - return true - } - case *TypeSpec: - if s.Name.IsExported() { - typeExports(s.Type) - return true - } + return filterType(t.Value, f) } return false } -func specListExports(list []Spec) []Spec { - j := 0 - for _, s := range list { - if specExports(s) { - list[j] = s - j++ - } - } - return list[0:j] -} - -func declExports(decl Decl) bool { - switch d := decl.(type) { - case *GenDecl: - d.Specs = specListExports(d.Specs) - return len(d.Specs) > 0 - case *FuncDecl: - d.Body = nil // strip body - return d.Name.IsExported() - } - return false -} - -// FileExports trims the AST for a Go source file in place such that only -// exported nodes remain: all top-level identifiers which are not exported -// and their associated information (such as type, initial value, or function -// body) are removed. Non-exported fields and methods of exported types are -// stripped, and the function bodies of exported functions are set to nil. -// The File.comments list is not changed. -// -// FileExports returns true if there is an exported declaration; it returns -// false otherwise. -// -func FileExports(src *File) bool { - j := 0 - for _, d := range src.Decls { - if declExports(d) { - src.Decls[j] = d - j++ - } - } - src.Decls = src.Decls[0:j] - return j > 0 -} - -// PackageExports trims the AST for a Go package in place such that only -// exported nodes remain. The pkg.Files list is not changed, so that file -// names and top-level package comments don't get lost. -// -// PackageExports returns true if there is an exported declaration; it -// returns false otherwise. -// -func PackageExports(pkg *Package) bool { - hasExports := false - for _, f := range pkg.Files { - if FileExports(f) { - hasExports = true - } - } - return hasExports -} - -// ---------------------------------------------------------------------------- -// General filtering - -type Filter func(string) bool - -func filterIdentList(list []*Ident, f Filter) []*Ident { - j := 0 - for _, x := range list { - if f(x.Name) { - list[j] = x - j++ - } - } - return list[0:j] -} - -func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { - if fields == nil { - return false - } - list := fields.List - j := 0 - for _, f := range list { - keepField := false - if len(f.Names) == 0 { - // anonymous field - name := fieldName(f.Type) - keepField = name != nil && filter(name.Name) - } else { - n := len(f.Names) - f.Names = filterIdentList(f.Names, filter) - if len(f.Names) < n { - removedFields = true - } - keepField = len(f.Names) > 0 - } - if keepField { - list[j] = f - j++ - } - } - if j < len(list) { - removedFields = true - } - fields.List = list[0:j] - return -} - func filterSpec(spec Spec, f Filter) bool { switch s := spec.(type) { case *ValueSpec: s.Names = filterIdentList(s.Names, f) - return len(s.Names) > 0 + if len(s.Names) > 0 { + if f == exportFilter { + filterType(s.Type, f) + } + return true + } case *TypeSpec: if f(s.Name.Name) { + if f == exportFilter { + filterType(s.Type, f) + } return true } - switch t := s.Type.(type) { - case *StructType: - if filterFieldList(t.Fields, f) { - t.Incomplete = true - } - return len(t.Fields.List) > 0 - case *InterfaceType: - if filterFieldList(t.Methods, f) { - t.Incomplete = true - } - return len(t.Methods.List) > 0 + if f != exportFilter { + // For general filtering (not just exports), + // filter type even if name is not filtered + // out. + // If the type contains filtered elements, + // keep the declaration. + return filterType(s.Type, f) } } return false @@ -293,8 +214,8 @@ func FilterDecl(decl Decl, f Filter) bool { // names from top-level declarations (including struct field and // interface method names, but not from parameter lists) that don't // pass through the filter f. If the declaration is empty afterwards, -// the declaration is removed from the AST. -// The File.comments list is not changed. +// the declaration is removed from the AST. The File.Comments list +// is not changed. // // FilterFile returns true if there are any top-level declarations // left after filtering; it returns false otherwise. @@ -311,13 +232,13 @@ func FilterFile(src *File, f Filter) bool { return j > 0 } -// FilterPackage trims the AST for a Go package in place by removing all -// names from top-level declarations (including struct field and +// FilterPackage trims the AST for a Go package in place by removing +// all names from top-level declarations (including struct field and // interface method names, but not from parameter lists) that don't // pass through the filter f. If the declaration is empty afterwards, -// the declaration is removed from the AST. -// The pkg.Files list is not changed, so that file names and top-level -// package comments don't get lost. +// the declaration is removed from the AST. The pkg.Files list is not +// changed, so that file names and top-level package comments don't get +// lost. // // FilterPackage returns true if there are any top-level declarations // left after filtering; it returns false otherwise. @@ -344,6 +265,8 @@ const ( // If set, comments that are not associated with a specific // AST node (as Doc or Comment) are excluded. FilterUnassociatedComments + // If set, duplicate import declarations are excluded. + FilterImportDuplicates ) // separator is an empty //-style comment that is interspersed between @@ -459,6 +382,31 @@ func MergePackageFiles(pkg *Package, mode MergeMode) *File { } } + // Collect import specs from all package files. + var imports []*ImportSpec + if mode&FilterImportDuplicates != 0 { + seen := make(map[string]bool) + for _, f := range pkg.Files { + for _, imp := range f.Imports { + if path := imp.Path.Value; !seen[path] { + // TODO: consider handling cases where: + // - 2 imports exist with the same import path but + // have different local names (one should probably + // keep both of them) + // - 2 imports exist but only one has a comment + // - 2 imports exist and they both have (possibly + // different) comments + imports = append(imports, imp) + seen[path] = true + } + } + } + } else { + for _, f := range pkg.Files { + imports = append(imports, f.Imports...) + } + } + // Collect comments from all package files. var comments []*CommentGroup if mode&FilterUnassociatedComments == 0 { @@ -469,7 +417,6 @@ func MergePackageFiles(pkg *Package, mode MergeMode) *File { } } - // TODO(gri) need to compute pkgScope and unresolved identifiers! - // TODO(gri) need to compute imports! - return &File{doc, pos, NewIdent(pkg.Name), decls, nil, nil, nil, comments} + // TODO(gri) need to compute unresolved identifiers! + return &File{doc, pos, NewIdent(pkg.Name), decls, pkg.Scope, imports, nil, comments} } diff --git a/libgo/go/go/build/build_test.go b/libgo/go/go/build/build_test.go index e59d876..68a4180 100644 --- a/libgo/go/go/build/build_test.go +++ b/libgo/go/go/build/build_test.go @@ -7,44 +7,84 @@ package build import ( "exec" "path/filepath" + "reflect" + "runtime" + "sort" "testing" ) -var buildPkgs = []string{ - "go/build/pkgtest", - "go/build/cmdtest", - "go/build/cgotest", +func sortstr(x []string) []string { + sort.Strings(x) + return x +} + +var buildPkgs = []struct { + dir string + info *DirInfo +}{ + { + "go/build/pkgtest", + &DirInfo{ + GoFiles: []string{"pkgtest.go"}, + SFiles: []string{"sqrt_" + runtime.GOARCH + ".s"}, + Package: "pkgtest", + Imports: []string{"os"}, + TestImports: []string{"fmt", "pkgtest"}, + TestGoFiles: sortstr([]string{"sqrt_test.go", "sqrt_" + runtime.GOARCH + "_test.go"}), + XTestGoFiles: []string{"xsqrt_test.go"}, + }, + }, + { + "go/build/cmdtest", + &DirInfo{ + GoFiles: []string{"main.go"}, + Package: "main", + Imports: []string{"go/build/pkgtest"}, + }, + }, + { + "go/build/cgotest", + &DirInfo{ + CgoFiles: []string{"cgotest.go"}, + CFiles: []string{"cgotest.c"}, + Imports: []string{"C", "unsafe"}, + Package: "cgotest", + }, + }, } const cmdtestOutput = "3" func TestBuild(t *testing.T) { - for _, pkg := range buildPkgs { + for _, tt := range buildPkgs { tree := Path[0] // Goroot - dir := filepath.Join(tree.SrcDir(), pkg) - - info, err := ScanDir(dir, true) + dir := filepath.Join(tree.SrcDir(), tt.dir) + info, err := ScanDir(dir) if err != nil { - t.Error("ScanDir:", err) + t.Errorf("ScanDir(%#q): %v", tt.dir, err) + continue + } + if !reflect.DeepEqual(info, tt.info) { + t.Errorf("ScanDir(%#q) = %#v, want %#v\n", tt.dir, info, tt.info) continue } - s, err := Build(tree, pkg, info) + s, err := Build(tree, tt.dir, info) if err != nil { - t.Error("Build:", err) + t.Errorf("Build(%#q): %v", tt.dir, err) continue } if err := s.Run(); err != nil { - t.Error("Run:", err) + t.Errorf("Run(%#q): %v", tt.dir, err) continue } - if pkg == "go/build/cmdtest" { + if tt.dir == "go/build/cmdtest" { bin := s.Output[0] b, err := exec.Command(bin).CombinedOutput() if err != nil { - t.Errorf("exec: %s: %v", bin, err) + t.Errorf("exec %s: %v", bin, err) continue } if string(b) != cmdtestOutput { @@ -52,6 +92,7 @@ func TestBuild(t *testing.T) { } } + // Deferred because cmdtest depends on pkgtest. defer func(s *Script) { if err := s.Nuke(); err != nil { t.Errorf("nuking: %v", err) diff --git a/libgo/go/go/build/dir.go b/libgo/go/go/build/dir.go index e0000b5..3ee10ab 100644 --- a/libgo/go/go/build/dir.go +++ b/libgo/go/go/build/dir.go @@ -5,168 +5,559 @@ package build import ( + "bytes" + "fmt" + "go/ast" + "go/doc" "go/parser" "go/token" + "io/ioutil" "log" "os" + "path" "path/filepath" + "runtime" + "sort" "strconv" "strings" - "runtime" + "unicode" ) +// A Context specifies the supporting context for a build. +type Context struct { + GOARCH string // target architecture + GOOS string // target operating system + // TODO(rsc,adg): GOPATH + + // By default, ScanDir uses the operating system's + // file system calls to read directories and files. + // Callers can override those calls to provide other + // ways to read data by setting ReadDir and ReadFile. + // ScanDir does not make any assumptions about the + // format of the strings dir and file: they can be + // slash-separated, backslash-separated, even URLs. + + // ReadDir returns a slice of *os.FileInfo, sorted by Name, + // describing the content of the named directory. + // The dir argument is the argument to ScanDir. + // If ReadDir is nil, ScanDir uses io.ReadDir. + ReadDir func(dir string) (fi []*os.FileInfo, err os.Error) + + // ReadFile returns the content of the file named file + // in the directory named dir. The dir argument is the + // argument to ScanDir, and the file argument is the + // Name field from an *os.FileInfo returned by ReadDir. + // The returned path is the full name of the file, to be + // used in error messages. + // + // If ReadFile is nil, ScanDir uses filepath.Join(dir, file) + // as the path and ioutil.ReadFile to read the data. + ReadFile func(dir, file string) (path string, content []byte, err os.Error) +} + +func (ctxt *Context) readDir(dir string) ([]*os.FileInfo, os.Error) { + if f := ctxt.ReadDir; f != nil { + return f(dir) + } + return ioutil.ReadDir(dir) +} + +func (ctxt *Context) readFile(dir, file string) (string, []byte, os.Error) { + if f := ctxt.ReadFile; f != nil { + return f(dir, file) + } + p := filepath.Join(dir, file) + content, err := ioutil.ReadFile(p) + return p, content, err +} + +// The DefaultContext is the default Context for builds. +// It uses the GOARCH and GOOS environment variables +// if set, or else the compiled code's GOARCH and GOOS. +var DefaultContext = Context{ + GOARCH: envOr("GOARCH", runtime.GOARCH), + GOOS: envOr("GOOS", runtime.GOOS), +} + +func envOr(name, def string) string { + s := os.Getenv(name) + if s == "" { + return def + } + return s +} + type DirInfo struct { + Package string // Name of package in dir + PackageComment *ast.CommentGroup // Package comments from GoFiles + ImportPath string // Import path of package in dir + Imports []string // All packages imported by GoFiles + + // Source files GoFiles []string // .go files in dir (excluding CgoFiles) - CgoFiles []string // .go files that import "C" CFiles []string // .c files in dir SFiles []string // .s files in dir - Imports []string // All packages imported by goFiles - PkgName string // Name of package in dir + CgoFiles []string // .go files that import "C" + + // Cgo directives + CgoPkgConfig []string // Cgo pkg-config directives + CgoCFLAGS []string // Cgo CFLAGS directives + CgoLDFLAGS []string // Cgo LDFLAGS directives + + // Test information + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package + TestImports []string // All packages imported by (X)TestGoFiles } func (d *DirInfo) IsCommand() bool { - return d.PkgName == "main" + // TODO(rsc): This is at least a little bogus. + return d.Package == "main" +} + +// ScanDir calls DefaultContext.ScanDir. +func ScanDir(dir string) (info *DirInfo, err os.Error) { + return DefaultContext.ScanDir(dir) } // ScanDir returns a structure with details about the Go content found // in the given directory. The file lists exclude: // -// - files in package main (unless allowMain is true) +// - files in package main (unless no other package is found) // - files in package documentation // - files ending in _test.go -// - files starting with _ or . +// - files starting with _ or . // -// Only files that satisfy the goodOSArch function are included. -func ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) { - f, err := os.Open(dir) - if err != nil { - return nil, err - } - dirs, err := f.Readdir(-1) - f.Close() +func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { + dirs, err := ctxt.readDir(dir) if err != nil { return nil, err } var di DirInfo imported := make(map[string]bool) + testImported := make(map[string]bool) fset := token.NewFileSet() - for i := range dirs { - d := &dirs[i] + for _, d := range dirs { + if !d.IsRegular() { + continue + } if strings.HasPrefix(d.Name, "_") || strings.HasPrefix(d.Name, ".") { continue } - if !goodOSArch(d.Name) { + if !ctxt.goodOSArchFile(d.Name) { continue } - switch filepath.Ext(d.Name) { - case ".go": - if strings.HasSuffix(d.Name, "_test.go") { - continue - } + ext := path.Ext(d.Name) + switch ext { + case ".go", ".c", ".s": + // tentatively okay + default: + // skip + continue + } + + // Look for +build comments to accept or reject the file. + filename, data, err := ctxt.readFile(dir, d.Name) + if err != nil { + return nil, err + } + if !ctxt.shouldBuild(data) { + continue + } + + // Going to save the file. For non-Go files, can stop here. + switch ext { case ".c": di.CFiles = append(di.CFiles, d.Name) continue case ".s": di.SFiles = append(di.SFiles, d.Name) continue - default: - continue } - filename := filepath.Join(dir, d.Name) - pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly) + pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly|parser.ParseComments) if err != nil { return nil, err } - s := string(pf.Name.Name) - if s == "main" && !allowMain { + + pkg := string(pf.Name.Name) + if pkg == "main" && di.Package != "" && di.Package != "main" { continue } - if s == "documentation" { + if pkg == "documentation" { continue } - if di.PkgName == "" { - di.PkgName = s - } else if di.PkgName != s { - // Only if all files in the directory are in package main - // do we return PkgName=="main". - // A mix of main and another package reverts - // to the original (allowMain=false) behaviour. - if s == "main" || di.PkgName == "main" { - return ScanDir(dir, false) + + isTest := strings.HasSuffix(d.Name, "_test.go") + if isTest && strings.HasSuffix(pkg, "_test") { + pkg = pkg[:len(pkg)-len("_test")] + } + + if pkg != di.Package && di.Package == "main" { + // Found non-main package but was recording + // information about package main. Reset. + di = DirInfo{} + } + if di.Package == "" { + di.Package = pkg + } else if pkg != di.Package { + return nil, fmt.Errorf("%s: found packages %s and %s", dir, pkg, di.Package) + } + if pf.Doc != nil { + if di.PackageComment != nil { + di.PackageComment.List = append(di.PackageComment.List, pf.Doc.List...) + } else { + di.PackageComment = pf.Doc } - return nil, os.NewError("multiple package names in " + dir) } + + // Record imports and information about cgo. isCgo := false - for _, spec := range pf.Imports { - quoted := string(spec.Path.Value) - path, err := strconv.Unquote(quoted) - if err != nil { - log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) + for _, decl := range pf.Decls { + d, ok := decl.(*ast.GenDecl) + if !ok { + continue } - imported[path] = true - if path == "C" { - isCgo = true + for _, dspec := range d.Specs { + spec, ok := dspec.(*ast.ImportSpec) + if !ok { + continue + } + quoted := string(spec.Path.Value) + path, err := strconv.Unquote(quoted) + if err != nil { + log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) + } + if isTest { + testImported[path] = true + } else { + imported[path] = true + } + if path == "C" { + if isTest { + return nil, fmt.Errorf("%s: use of cgo in test not supported", filename) + } + cg := spec.Doc + if cg == nil && len(d.Specs) == 1 { + cg = d.Doc + } + if cg != nil { + if err := ctxt.saveCgo(filename, &di, cg); err != nil { + return nil, err + } + } + isCgo = true + } } } if isCgo { di.CgoFiles = append(di.CgoFiles, d.Name) + } else if isTest { + if pkg == string(pf.Name.Name) { + di.TestGoFiles = append(di.TestGoFiles, d.Name) + } else { + di.XTestGoFiles = append(di.XTestGoFiles, d.Name) + } } else { di.GoFiles = append(di.GoFiles, d.Name) } } + if di.Package == "" { + return nil, fmt.Errorf("%s: no Go source files", dir) + } di.Imports = make([]string, len(imported)) i := 0 for p := range imported { di.Imports[i] = p i++ } + di.TestImports = make([]string, len(testImported)) + i = 0 + for p := range testImported { + di.TestImports[i] = p + i++ + } + // File name lists are sorted because ReadDir sorts. + sort.Strings(di.Imports) + sort.Strings(di.TestImports) return &di, nil } -// goodOSArch returns false if the filename contains a $GOOS or $GOARCH +var slashslash = []byte("//") +var plusBuild = []byte("+build") + +// shouldBuild reports whether it is okay to use this file, +// The rule is that in the file's leading run of // comments +// and blank lines, which must be followed by a blank line +// (to avoid including a Go package clause doc comment), +// lines beginning with '// +build' are taken as build directives. +// +// The file is accepted only if each such line lists something +// matching the file. For example: +// +// // +build windows linux +// +// marks the file as applicable only on Windows and Linux. +// +func (ctxt *Context) shouldBuild(content []byte) bool { + // Pass 1. Identify leading run of // comments and blank lines, + // which must be followed by a blank line. + end := 0 + p := content + for len(p) > 0 { + line := p + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, p = line[:i], p[i+1:] + } else { + p = p[len(p):] + } + line = bytes.TrimSpace(line) + if len(line) == 0 { // Blank line + end = cap(content) - cap(line) // &line[0] - &content[0] + continue + } + if !bytes.HasPrefix(line, slashslash) { // Not comment line + break + } + } + content = content[:end] + + // Pass 2. Process each line in the run. + p = content + for len(p) > 0 { + line := p + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, p = line[:i], p[i+1:] + } else { + p = p[len(p):] + } + line = bytes.TrimSpace(line) + if bytes.HasPrefix(line, slashslash) { + line = bytes.TrimSpace(line[len(slashslash):]) + if len(line) > 0 && line[0] == '+' { + // Looks like a comment +line. + f := strings.Fields(string(line)) + if f[0] == "+build" { + ok := false + for _, tok := range f[1:] { + if ctxt.matchOSArch(tok) { + ok = true + break + } + } + if !ok { + return false // this one doesn't match + } + } + } + } + } + return true // everything matches +} + +// saveCgo saves the information from the #cgo lines in the import "C" comment. +// These lines set CFLAGS and LDFLAGS and pkg-config directives that affect +// the way cgo's C code is built. +// +// TODO(rsc): This duplicates code in cgo. +// Once the dust settles, remove this code from cgo. +func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) os.Error { + text := doc.CommentText(cg) + for _, line := range strings.Split(text, "\n") { + orig := line + + // Line is + // #cgo [GOOS/GOARCH...] LDFLAGS: stuff + // + line = strings.TrimSpace(line) + if len(line) < 5 || line[:4] != "#cgo" || (line[4] != ' ' && line[4] != '\t') { + continue + } + + // Split at colon. + line = strings.TrimSpace(line[4:]) + i := strings.Index(line, ":") + if i < 0 { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + line, argstr := line[:i], line[i+1:] + + // Parse GOOS/GOARCH stuff. + f := strings.Fields(line) + if len(f) < 1 { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + + cond, verb := f[:len(f)-1], f[len(f)-1] + if len(cond) > 0 { + ok := false + for _, c := range cond { + if ctxt.matchOSArch(c) { + ok = true + break + } + } + if !ok { + continue + } + } + + args, err := splitQuoted(argstr) + if err != nil { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + for _, arg := range args { + if !safeName(arg) { + return fmt.Errorf("%s: malformed #cgo argument: %s", filename, arg) + } + } + + switch verb { + case "CFLAGS": + di.CgoCFLAGS = append(di.CgoCFLAGS, args...) + case "LDFLAGS": + di.CgoLDFLAGS = append(di.CgoLDFLAGS, args...) + case "pkg-config": + di.CgoPkgConfig = append(di.CgoPkgConfig, args...) + default: + return fmt.Errorf("%s: invalid #cgo verb: %s", filename, orig) + } + } + return nil +} + +var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +func safeName(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 { + return false + } + } + return true +} + +// splitQuoted splits the string s around each instance of one or more consecutive +// white space characters while taking into account quotes and escaping, and +// returns an array of substrings of s or an empty list if s contains only white space. +// Single quotes and double quotes are recognized to prevent splitting within the +// quoted region, and are removed from the resulting substrings. If a quote in s +// isn't closed err will be set and r will have the unclosed argument as the +// last element. The backslash is used for escaping. +// +// For example, the following string: +// +// a b:"c d" 'e''f' "g\"" +// +// Would be parsed as: +// +// []string{"a", "b:c d", "ef", `g"`} +// +func splitQuoted(s string) (r []string, err os.Error) { + var args []string + arg := make([]int, len(s)) + escaped := false + quoted := false + quote := 0 + i := 0 + for _, rune := range s { + switch { + case escaped: + escaped = false + case rune == '\\': + escaped = true + continue + case quote != 0: + if rune == quote { + quote = 0 + continue + } + case rune == '"' || rune == '\'': + quoted = true + quote = rune + continue + case unicode.IsSpace(rune): + if quoted || i > 0 { + quoted = false + args = append(args, string(arg[:i])) + i = 0 + } + continue + } + arg[i] = rune + i++ + } + if quoted || i > 0 { + args = append(args, string(arg[:i])) + } + if quote != 0 { + err = os.NewError("unclosed quote") + } else if escaped { + err = os.NewError("unfinished escaping") + } + return args, err +} + +// matchOSArch returns true if the name is one of: +// +// $GOOS +// $GOARCH +// $GOOS/$GOARCH +// +func (ctxt *Context) matchOSArch(name string) bool { + if name == ctxt.GOOS || name == ctxt.GOARCH { + return true + } + i := strings.Index(name, "/") + return i >= 0 && name[:i] == ctxt.GOOS && name[i+1:] == ctxt.GOARCH +} + +// goodOSArchFile returns false if the name contains a $GOOS or $GOARCH // suffix which does not match the current system. -// The recognized filename formats are: +// The recognized name formats are: // // name_$(GOOS).* // name_$(GOARCH).* // name_$(GOOS)_$(GOARCH).* +// name_$(GOOS)_test.* +// name_$(GOARCH)_test.* +// name_$(GOOS)_$(GOARCH)_test.* // -func goodOSArch(filename string) bool { - if dot := strings.Index(filename, "."); dot != -1 { - filename = filename[:dot] +func (ctxt *Context) goodOSArchFile(name string) bool { + if dot := strings.Index(name, "."); dot != -1 { + name = name[:dot] + } + l := strings.Split(name, "_") + if n := len(l); n > 0 && l[n-1] == "test" { + l = l[:n-1] } - l := strings.Split(filename, "_") n := len(l) - if n == 0 { - return true + if n >= 2 && knownOS[l[n-2]] && knownArch[l[n-1]] { + return l[n-2] == ctxt.GOOS && l[n-1] == ctxt.GOARCH } - if good, known := goodOS[l[n-1]]; known { - return good + if n >= 1 && knownOS[l[n-1]] { + return l[n-1] == ctxt.GOOS } - if good, known := goodArch[l[n-1]]; known { - if !good || n < 2 { - return false - } - good, known = goodOS[l[n-2]] - return good || !known + if n >= 1 && knownArch[l[n-1]] { + return l[n-1] == ctxt.GOARCH } return true } -var goodOS = make(map[string]bool) -var goodArch = make(map[string]bool) +var knownOS = make(map[string]bool) +var knownArch = make(map[string]bool) func init() { - goodOS = make(map[string]bool) - goodArch = make(map[string]bool) for _, v := range strings.Fields(goosList) { - goodOS[v] = v == runtime.GOOS + knownOS[v] = true } for _, v := range strings.Fields(goarchList) { - goodArch[v] = v == runtime.GOARCH + knownArch[v] = true } } diff --git a/libgo/go/go/build/pkgtest/pkgtest.go b/libgo/go/go/build/pkgtest/pkgtest.go index 9322f5e..03ebb98 100644 --- a/libgo/go/go/build/pkgtest/pkgtest.go +++ b/libgo/go/go/build/pkgtest/pkgtest.go @@ -4,6 +4,10 @@ package pkgtest -func Foo() {} +import "os" + +func Foo() os.Error { + return nil +} func Sqrt(x float64) float64 diff --git a/libgo/go/go/build/pkgtest/sqrt_386_test.go b/libgo/go/go/build/pkgtest/sqrt_386_test.go new file mode 100644 index 0000000..26b483f --- /dev/null +++ b/libgo/go/go/build/pkgtest/sqrt_386_test.go @@ -0,0 +1 @@ +package pkgtest diff --git a/libgo/go/go/build/pkgtest/sqrt_amd64_test.go b/libgo/go/go/build/pkgtest/sqrt_amd64_test.go new file mode 100644 index 0000000..26b483f --- /dev/null +++ b/libgo/go/go/build/pkgtest/sqrt_amd64_test.go @@ -0,0 +1 @@ +package pkgtest diff --git a/libgo/go/go/build/pkgtest/sqrt_arm_test.go b/libgo/go/go/build/pkgtest/sqrt_arm_test.go new file mode 100644 index 0000000..26b483f --- /dev/null +++ b/libgo/go/go/build/pkgtest/sqrt_arm_test.go @@ -0,0 +1 @@ +package pkgtest diff --git a/libgo/go/go/build/pkgtest/sqrt_test.go b/libgo/go/go/build/pkgtest/sqrt_test.go new file mode 100644 index 0000000..95fb625 --- /dev/null +++ b/libgo/go/go/build/pkgtest/sqrt_test.go @@ -0,0 +1,5 @@ +package pkgtest + +import "fmt" + +var _ = fmt.Printf diff --git a/libgo/go/go/build/pkgtest/xsqrt_test.go b/libgo/go/go/build/pkgtest/xsqrt_test.go new file mode 100644 index 0000000..77e903d --- /dev/null +++ b/libgo/go/go/build/pkgtest/xsqrt_test.go @@ -0,0 +1,5 @@ +package pkgtest_test + +import "pkgtest" + +var _ = pkgtest.Foo diff --git a/libgo/go/go/build/syslist_test.go b/libgo/go/go/build/syslist_test.go index eb0e5dc..d27630d 100644 --- a/libgo/go/go/build/syslist_test.go +++ b/libgo/go/go/build/syslist_test.go @@ -55,8 +55,8 @@ var tests = []GoodFileTest{ func TestGoodOSArch(t *testing.T) { for _, test := range tests { - if goodOSArch(test.name) != test.result { - t.Fatalf("goodOSArch(%q) != %v", test.name, test.result) + if DefaultContext.goodOSArchFile(test.name) != test.result { + t.Fatalf("goodOSArchFile(%q) != %v", test.name, test.result) } } } diff --git a/libgo/go/go/doc/example.go b/libgo/go/go/doc/example.go new file mode 100644 index 0000000..7fdf0bc --- /dev/null +++ b/libgo/go/go/doc/example.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Extract example functions from package ASTs. + +package doc + +import ( + "go/ast" + "strings" + "unicode" + "utf8" +) + +type Example struct { + Name string // name of the item being demonstrated + Body *ast.BlockStmt // code + Output string // expected output +} + +func Examples(pkg *ast.Package) []*Example { + var examples []*Example + for _, src := range pkg.Files { + for _, decl := range src.Decls { + f, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + name := f.Name.Name + if !isTest(name, "Example") { + continue + } + examples = append(examples, &Example{ + Name: name[len("Example"):], + Body: f.Body, + Output: CommentText(f.Doc), + }) + } + } + return examples +} + +// isTest tells whether name looks like a test, example, or benchmark. +// It is a Test (say) if there is a character after Test that is not a +// lower-case letter. (We don't want Testiness.) +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true + } + rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(rune) +} diff --git a/libgo/go/go/parser/parser.go b/libgo/go/go/parser/parser.go index 9c14d16..be82b2f 100644 --- a/libgo/go/go/parser/parser.go +++ b/libgo/go/go/parser/parser.go @@ -587,7 +587,6 @@ func (p *parser) parseStructType() *ast.StructType { } rbrace := p.expect(token.RBRACE) - // TODO(gri): store struct scope in AST return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false} } @@ -611,9 +610,6 @@ func (p *parser) tryVarType(isParam bool) ast.Expr { p.error(pos, "'...' parameter is missing type") typ = &ast.BadExpr{pos, p.pos} } - if p.tok != token.RPAREN { - p.error(pos, "can use '...' with last parameter type only") - } return &ast.Ellipsis{pos, typ} } return p.tryIdentOrType(false) @@ -636,21 +632,21 @@ func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) { } // a list of identifiers looks like a list of type names - for { - // parseVarType accepts any type (including parenthesized ones) - // even though the syntax does not permit them here: we - // accept them all for more robust parsing and complain - // afterwards - list = append(list, p.parseVarType(isParam)) + // + // parse/tryVarType accepts any type (including parenthesized + // ones) even though the syntax does not permit them here: we + // accept them all for more robust parsing and complain later + for typ := p.parseVarType(isParam); typ != nil; { + list = append(list, typ) if p.tok != token.COMMA { break } p.next() + typ = p.tryVarType(isParam) // maybe nil as in: func f(int,) {} } // if we had a list of identifiers, it must be followed by a type - typ = p.tryVarType(isParam) - if typ != nil { + if typ = p.tryVarType(isParam); typ != nil { p.resolve(typ) } @@ -800,7 +796,6 @@ func (p *parser) parseInterfaceType() *ast.InterfaceType { } rbrace := p.expect(token.RBRACE) - // TODO(gri): store interface scope in AST return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false} } @@ -1436,14 +1431,14 @@ func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) { case token.ARROW: // send statement arrow := p.pos - p.next() // consume "<-" + p.next() y := p.parseRhs() return &ast.SendStmt{x[0], arrow, y}, false case token.INC, token.DEC: // increment or decrement s := &ast.IncDecStmt{x[0], p.pos, p.tok} - p.next() // consume "++" or "--" + p.next() return s, false } @@ -1591,7 +1586,7 @@ func (p *parser) parseTypeList() (list []ast.Expr) { return } -func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause { +func (p *parser) parseCaseClause(typeSwitch bool) *ast.CaseClause { if p.trace { defer un(trace(p, "CaseClause")) } @@ -1600,10 +1595,10 @@ func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause { var list []ast.Expr if p.tok == token.CASE { p.next() - if exprSwitch { - list = p.parseRhsList() - } else { + if typeSwitch { list = p.parseTypeList() + } else { + list = p.parseRhsList() } } else { p.expect(token.DEFAULT) @@ -1617,15 +1612,19 @@ func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause { return &ast.CaseClause{pos, list, colon, body} } -func isExprSwitch(s ast.Stmt) bool { - if s == nil { - return true - } - if e, ok := s.(*ast.ExprStmt); ok { - if a, ok := e.X.(*ast.TypeAssertExpr); ok { - return a.Type != nil // regular type assertion - } - return true +func isTypeSwitchAssert(x ast.Expr) bool { + a, ok := x.(*ast.TypeAssertExpr) + return ok && a.Type == nil +} + +func isTypeSwitchGuard(s ast.Stmt) bool { + switch t := s.(type) { + case *ast.ExprStmt: + // x.(nil) + return isTypeSwitchAssert(t.X) + case *ast.AssignStmt: + // v := x.(nil) + return len(t.Lhs) == 1 && t.Tok == token.DEFINE && len(t.Rhs) == 1 && isTypeSwitchAssert(t.Rhs[0]) } return false } @@ -1651,28 +1650,41 @@ func (p *parser) parseSwitchStmt() ast.Stmt { s1 = s2 s2 = nil if p.tok != token.LBRACE { + // A TypeSwitchGuard may declare a variable in addition + // to the variable declared in the initial SimpleStmt. + // Introduce extra scope to avoid redeclaration errors: + // + // switch t := 0; t := x.(T) { ... } + // + // (this code is not valid Go because the first t will + // cannot be accessed and thus is never used, the extra + // scope is needed for the correct error message). + // + // If we don't have a type switch, s2 must be an expression. + // Having the extra nested but empty scope won't affect it. + p.openScope() + defer p.closeScope() s2, _ = p.parseSimpleStmt(basic) } } p.exprLev = prevLev } - exprSwitch := isExprSwitch(s2) + typeSwitch := isTypeSwitchGuard(s2) lbrace := p.expect(token.LBRACE) var list []ast.Stmt for p.tok == token.CASE || p.tok == token.DEFAULT { - list = append(list, p.parseCaseClause(exprSwitch)) + list = append(list, p.parseCaseClause(typeSwitch)) } rbrace := p.expect(token.RBRACE) p.expectSemi() body := &ast.BlockStmt{lbrace, list, rbrace} - if exprSwitch { - return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body} + if typeSwitch { + return &ast.TypeSwitchStmt{pos, s1, s2, body} } - // type switch - // TODO(gri): do all the checks! - return &ast.TypeSwitchStmt{pos, s1, s2, body} + + return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body} } func (p *parser) parseCommClause() *ast.CommClause { @@ -2001,14 +2013,12 @@ func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { defer un(trace(p, "Receiver")) } - pos := p.pos par := p.parseParameters(scope, false) // must have exactly one receiver if par.NumFields() != 1 { - p.errorExpected(pos, "exactly one receiver") - // TODO determine a better range for BadExpr below - par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}} + p.errorExpected(par.Opening, "exactly one receiver") + par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{par.Opening, par.Closing + 1}}} return par } diff --git a/libgo/go/go/parser/parser_test.go b/libgo/go/go/parser/parser_test.go index 39a78e5..9705dcf 100644 --- a/libgo/go/go/parser/parser_test.go +++ b/libgo/go/go/parser/parser_test.go @@ -26,6 +26,9 @@ var illegalInputs = []interface{}{ `package p; func f() { for _ = range x ; ; {} };`, `package p; func f() { for ; ; _ = range x {} };`, `package p; func f() { for ; _ = range x ; {} };`, + `package p; func f() { switch t = t.(type) {} };`, + `package p; func f() { switch t, t = t.(type) {} };`, + `package p; func f() { switch t = t.(type), t {} };`, `package p; var a = [1]int; /* illegal expression */`, `package p; var a = [...]int; /* illegal expression */`, `package p; var a = struct{} /* illegal expression */`, @@ -61,6 +64,9 @@ var validPrograms = []interface{}{ `package p; func f(...T);`, `package p; func f(float, ...int);`, `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, + `package p; func f(int,) {};`, + `package p; func f(...int,) {};`, + `package p; func f(x ...int,) {};`, `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, @@ -74,7 +80,7 @@ var validPrograms = []interface{}{ func TestParseValidPrograms(t *testing.T) { for _, src := range validPrograms { - _, err := ParseFile(fset, "", src, 0) + _, err := ParseFile(fset, "", src, SpuriousErrors) if err != nil { t.Errorf("ParseFile(%q): %v", src, err) } diff --git a/libgo/go/go/printer/nodes.go b/libgo/go/go/printer/nodes.go index 9cd975e..3645306 100644 --- a/libgo/go/go/printer/nodes.go +++ b/libgo/go/go/printer/nodes.go @@ -269,6 +269,7 @@ func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exp func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) { p.print(fields.Opening, token.LPAREN) if len(fields.List) > 0 { + ws := indent var prevLine, line int for i, par := range fields.List { if i > 0 { @@ -278,19 +279,30 @@ func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) { } else { line = p.fset.Position(par.Type.Pos()).Line } - if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) { + if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ws, true) { + ws = ignore *multiLine = true } else { p.print(blank) } } if len(par.Names) > 0 { - p.identList(par.Names, false, multiLine) + // Very subtle: If we indented before (ws == ignore), identList + // won't indent again. If we didn't (ws == indent), identList will + // indent if the identList spans multiple lines, and it will outdent + // again at the end (and still ws == indent). Thus, a subsequent indent + // by a linebreak call after a type, or in the next multi-line identList + // will do the right thing. + p.identList(par.Names, ws == indent, multiLine) p.print(blank) } p.expr(par.Type, multiLine) prevLine = p.fset.Position(par.Type.Pos()).Line } + if ws == ignore { + // unindent if we indented + p.print(unindent) + } } p.print(fields.Closing, token.RPAREN) } diff --git a/libgo/go/go/printer/printer.go b/libgo/go/go/printer/printer.go index 871fefa..bfabd74 100644 --- a/libgo/go/go/printer/printer.go +++ b/libgo/go/go/printer/printer.go @@ -13,7 +13,6 @@ import ( "io" "os" "path/filepath" - "runtime" "tabwriter" ) @@ -55,12 +54,17 @@ const ( noExtraLinebreak ) +// local error wrapper so we can distinguish os.Errors we want to return +// as errors from genuine panics (which we don't want to return as errors) +type osError struct { + err os.Error +} + type printer struct { // Configuration (does not change after initialization) output io.Writer Config - fset *token.FileSet - errors chan os.Error + fset *token.FileSet // Current state written int // number of bytes written @@ -95,7 +99,6 @@ func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet, nodeS p.output = output p.Config = *cfg p.fset = fset - p.errors = make(chan os.Error) p.wsbuf = make([]whiteSpace, 0, 16) // whitespace sequences are short p.nodeSizes = nodeSizes } @@ -143,8 +146,7 @@ func (p *printer) write0(data []byte) { n, err := p.output.Write(data) p.written += n if err != nil { - p.errors <- err - runtime.Goexit() + panic(osError{err}) } } } @@ -923,7 +925,7 @@ type Config struct { } // fprint implements Fprint and takes a nodesSizes map for setting up the printer state. -func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{}, nodeSizes map[ast.Node]int) (int, os.Error) { +func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{}, nodeSizes map[ast.Node]int) (written int, err os.Error) { // redirect output through a trimmer to eliminate trailing whitespace // (Input to a tabwriter must be untrimmed since trailing tabs provide // formatting information. The tabwriter could provide trimming @@ -950,47 +952,50 @@ func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{ output = tw } - // setup printer and print node + // setup printer var p printer p.init(output, cfg, fset, nodeSizes) - go func() { - switch n := node.(type) { - case ast.Expr: - p.useNodeComments = true - p.expr(n, ignoreMultiLine) - case ast.Stmt: - p.useNodeComments = true - // A labeled statement will un-indent to position the - // label. Set indent to 1 so we don't get indent "underflow". - if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt { - p.indent = 1 - } - p.stmt(n, false, ignoreMultiLine) - case ast.Decl: - p.useNodeComments = true - p.decl(n, ignoreMultiLine) - case ast.Spec: - p.useNodeComments = true - p.spec(n, 1, false, ignoreMultiLine) - case *ast.File: - p.comments = n.Comments - p.useNodeComments = n.Comments == nil - p.file(n) - default: - p.errors <- fmt.Errorf("printer.Fprint: unsupported node type %T", n) - runtime.Goexit() + defer func() { + written = p.written + if e := recover(); e != nil { + err = e.(osError).err // re-panics if it's not a local osError } - p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF) - p.errors <- nil // no errors }() - err := <-p.errors // wait for completion of goroutine + + // print node + switch n := node.(type) { + case ast.Expr: + p.useNodeComments = true + p.expr(n, ignoreMultiLine) + case ast.Stmt: + p.useNodeComments = true + // A labeled statement will un-indent to position the + // label. Set indent to 1 so we don't get indent "underflow". + if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt { + p.indent = 1 + } + p.stmt(n, false, ignoreMultiLine) + case ast.Decl: + p.useNodeComments = true + p.decl(n, ignoreMultiLine) + case ast.Spec: + p.useNodeComments = true + p.spec(n, 1, false, ignoreMultiLine) + case *ast.File: + p.comments = n.Comments + p.useNodeComments = n.Comments == nil + p.file(n) + default: + panic(osError{fmt.Errorf("printer.Fprint: unsupported node type %T", n)}) + } + p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF) // flush tabwriter, if any if tw != nil { tw.Flush() // ignore errors } - return p.written, err + return } // Fprint "pretty-prints" an AST node to output and returns the number diff --git a/libgo/go/go/printer/printer_test.go b/libgo/go/go/printer/printer_test.go index ff2d906..a644aa3 100644 --- a/libgo/go/go/printer/printer_test.go +++ b/libgo/go/go/printer/printer_test.go @@ -7,10 +7,10 @@ package printer import ( "bytes" "flag" - "io/ioutil" "go/ast" "go/parser" "go/token" + "io/ioutil" "path/filepath" "testing" "time" @@ -192,3 +192,15 @@ func TestLineComments(t *testing.T) { t.Errorf("got %d, expected %d\n", nlines, expected) } } + +// Verify that the printer can be invoked during initialization. +func init() { + const name = "foobar" + var buf bytes.Buffer + if err := Fprint(&buf, fset, &ast.Ident{Name: name}); err != nil { + panic(err) + } + if s := buf.String(); s != name { + panic("got " + s + ", want " + name) + } +} diff --git a/libgo/go/go/printer/testdata/declarations.golden b/libgo/go/go/printer/testdata/declarations.golden index 970533e..bfa2568 100644 --- a/libgo/go/go/printer/testdata/declarations.golden +++ b/libgo/go/go/printer/testdata/declarations.golden @@ -692,56 +692,119 @@ func _(x ...chan int) // these parameter lists must remain multi-line since they are multi-line in the source func _(bool, -int) { + int) { } func _(x bool, -y int) { + y int) { } func _(x, -y bool) { + y bool) { } func _(bool, // comment -int) { + int) { } func _(x bool, // comment -y int) { + y int) { } func _(x, // comment -y bool) { + y bool) { } func _(bool, // comment -// comment -int) { + // comment + int) { } func _(x bool, // comment -// comment -y int) { + // comment + y int) { } func _(x, // comment -// comment -y bool) { + // comment + y bool) { } func _(bool, -// comment -int) { + // comment + int) { } func _(x bool, -// comment -y int) { + // comment + y int) { } func _(x, -// comment -y bool) { + // comment + y bool) { } func _(x, // comment -y, // comment -z bool) { + y, // comment + z bool) { } func _(x, // comment -y, // comment -z bool) { + y, // comment + z bool) { } func _(x int, // comment -y float, // comment -z bool) { + y float, // comment + z bool) { +} + +// properly indent multi-line signatures +func ManageStatus(in <-chan *Status, req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int) { +} + +func MultiLineSignature0(a, b, c int) { +} + +func MultiLineSignature1(a, b, c int, + u, v, w float) { +} + +func MultiLineSignature2(a, b, + c int) { +} + +func MultiLineSignature3(a, b, + c int, u, v, + w float, + x ...int) { +} + +func MultiLineSignature4(a, b, c int, + u, v, + w float, + x ...int) { +} + +func MultiLineSignature5(a, b, c int, + u, v, w float, + p, q, + r string, + x ...int) { +} + +// make sure it also works for methods in interfaces +type _ interface { + MultiLineSignature0(a, b, c int) + + MultiLineSignature1(a, b, c int, + u, v, w float) + + MultiLineSignature2(a, b, + c int) + + MultiLineSignature3(a, b, + c int, u, v, + w float, + x ...int) + + MultiLineSignature4(a, b, c int, + u, v, + w float, + x ...int) + + MultiLineSignature5(a, b, c int, + u, v, w float, + p, q, + r string, + x ...int) } diff --git a/libgo/go/go/printer/testdata/declarations.input b/libgo/go/go/printer/testdata/declarations.input index c613409..1d69c57 100644 --- a/libgo/go/go/printer/testdata/declarations.input +++ b/libgo/go/go/printer/testdata/declarations.input @@ -755,3 +755,79 @@ func _(x int, // comment y float, // comment z bool) { } + + +// properly indent multi-line signatures +func ManageStatus(in <-chan *Status, req <-chan Request, +stat chan<- *TargetInfo, +TargetHistorySize int) { +} + +func MultiLineSignature0( +a, b, c int, +) {} + +func MultiLineSignature1( +a, b, c int, +u, v, w float, +) {} + +func MultiLineSignature2( +a, b, +c int, +) {} + +func MultiLineSignature3( +a, b, +c int, u, v, +w float, + x ...int) {} + +func MultiLineSignature4( +a, b, c int, +u, v, +w float, + x ...int) {} + +func MultiLineSignature5( +a, b, c int, +u, v, w float, +p, q, +r string, + x ...int) {} + +// make sure it also works for methods in interfaces +type _ interface { +MultiLineSignature0( +a, b, c int, +) + +MultiLineSignature1( +a, b, c int, +u, v, w float, +) + +MultiLineSignature2( +a, b, +c int, +) + +MultiLineSignature3( +a, b, +c int, u, v, +w float, + x ...int) + +MultiLineSignature4( +a, b, c int, +u, v, +w float, + x ...int) + +MultiLineSignature5( +a, b, c int, +u, v, w float, +p, q, +r string, + x ...int) +} diff --git a/libgo/go/go/scanner/scanner.go b/libgo/go/go/scanner/scanner.go index 7f3dd23..589ec68 100644 --- a/libgo/go/go/scanner/scanner.go +++ b/libgo/go/go/scanner/scanner.go @@ -90,9 +90,8 @@ func (S *Scanner) next() { // They control scanner behavior. // const ( - ScanComments = 1 << iota // return comments as COMMENT tokens - AllowIllegalChars // do not report an error for illegal chars - InsertSemis // automatically insert semicolons + ScanComments = 1 << iota // return comments as COMMENT tokens + InsertSemis // automatically insert semicolons ) // Init prepares the scanner S to tokenize the text src by setting the @@ -152,7 +151,7 @@ func (S *Scanner) interpretLineComment(text []byte) { filename = filepath.Join(S.dir, filename) } // update scanner position - S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line + S.file.AddLineInfo(S.lineOffset+len(text)+1, filename, line) // +len(text)+1 since comment applies to next line } } } @@ -652,9 +651,7 @@ scanAgain: case '|': tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR) default: - if S.mode&AllowIllegalChars == 0 { - S.error(offs, fmt.Sprintf("illegal character %#U", ch)) - } + S.error(offs, fmt.Sprintf("illegal character %#U", ch)) insertSemi = S.insertSemi // preserve insertSemi info } } diff --git a/libgo/go/go/scanner/scanner_test.go b/libgo/go/go/scanner/scanner_test.go index eb9e1cb..0c2cbe6 100644 --- a/libgo/go/go/scanner/scanner_test.go +++ b/libgo/go/go/scanner/scanner_test.go @@ -420,14 +420,14 @@ var lines = []string{ func TestSemis(t *testing.T) { for _, line := range lines { - checkSemi(t, line, AllowIllegalChars|InsertSemis) - checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments) + checkSemi(t, line, InsertSemis) + checkSemi(t, line, InsertSemis|ScanComments) // if the input ended in newlines, the input must tokenize the // same with or without those newlines for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- { - checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis) - checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments) + checkSemi(t, line[0:i], InsertSemis) + checkSemi(t, line[0:i], InsertSemis|ScanComments) } } } @@ -529,27 +529,6 @@ func TestInit(t *testing.T) { } } -func TestIllegalChars(t *testing.T) { - var s Scanner - - const src = "*?*$*@*" - file := fset.AddFile("", fset.Base(), len(src)) - s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars) - for offs, ch := range src { - pos, tok, lit := s.Scan() - if poffs := file.Offset(pos); poffs != offs { - t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs) - } - if tok == token.ILLEGAL && lit != string(ch) { - t.Errorf("bad token: got %s, expected %s", lit, string(ch)) - } - } - - if s.ErrorCount != 0 { - t.Errorf("found %d errors", s.ErrorCount) - } -} - func TestStdErrorHander(t *testing.T) { const src = "@\n" + // illegal character, cause an error "@ @\n" + // two errors on the same line diff --git a/libgo/go/go/token/position.go b/libgo/go/go/token/position.go index c559e19..8cf3dcd 100644 --- a/libgo/go/go/token/position.go +++ b/libgo/go/go/token/position.go @@ -86,9 +86,11 @@ func searchFiles(a []*File, x int) int { } func (s *FileSet) file(p Pos) *File { + // common case: p is in last file touched if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size { return f } + // p is not in last file touched - search all files if i := searchFiles(s.files, int(p)); i >= 0 { f := s.files[i] // f.base <= int(p) by definition of searchFiles @@ -123,10 +125,6 @@ func (f *File) position(p Pos) (pos Position) { // Position converts a Pos in the fileset into a general Position. func (s *FileSet) Position(p Pos) (pos Position) { if p != NoPos { - // TODO(gri) consider optimizing the case where p - // is in the last file added, or perhaps - // looked at - will eliminate one level - // of search s.mutex.RLock() if f := s.file(p); f != nil { pos = f.position(p) @@ -136,10 +134,14 @@ func (s *FileSet) Position(p Pos) (pos Position) { return } +// A lineInfo object describes alternative file and line number +// information (such as provided via a //line comment in a .go +// file) for a given file offset. type lineInfo struct { - offset int - filename string - line int + // fields are exported to make them accessible to gob + Offset int + Filename string + Line int } // AddLineInfo adds alternative file and line number information for @@ -152,7 +154,7 @@ type lineInfo struct { // func (f *File) AddLineInfo(offset int, filename string, line int) { f.set.mutex.Lock() - if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset < f.size { + if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size { f.infos = append(f.infos, lineInfo{offset, filename, line}) } f.set.mutex.Unlock() @@ -317,7 +319,7 @@ func searchInts(a []int, x int) int { } func searchLineInfos(a []lineInfo, x int) int { - return sort.Search(len(a), func(i int) bool { return a[i].offset > x }) - 1 + return sort.Search(len(a), func(i int) bool { return a[i].Offset > x }) - 1 } // info returns the file name, line, and column number for a file offset. @@ -330,9 +332,9 @@ func (f *File) info(offset int) (filename string, line, column int) { // almost no files have extra line infos if i := searchLineInfos(f.infos, offset); i >= 0 { alt := &f.infos[i] - filename = alt.filename - if i := searchInts(f.lines, alt.offset); i >= 0 { - line += alt.line - i - 1 + filename = alt.Filename + if i := searchInts(f.lines, alt.Offset); i >= 0 { + line += alt.Line - i - 1 } } } diff --git a/libgo/go/go/token/serialize.go b/libgo/go/go/token/serialize.go new file mode 100644 index 0000000..5fb47f7 --- /dev/null +++ b/libgo/go/go/token/serialize.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "gob" + "io" + "os" +) + +type serializedFile struct { + // fields correspond 1:1 to fields with same (lower-case) name in File + Name string + Base int + Size int + Lines []int + Infos []lineInfo +} + +type serializedFileSet struct { + Base int + Files []serializedFile +} + +func (s *serializedFileSet) Read(r io.Reader) os.Error { + return gob.NewDecoder(r).Decode(s) +} + +func (s *serializedFileSet) Write(w io.Writer) os.Error { + return gob.NewEncoder(w).Encode(s) +} + +// Read reads the fileset from r into s; s must not be nil. +// If r does not also implement io.ByteReader, it will be wrapped in a bufio.Reader. +func (s *FileSet) Read(r io.Reader) os.Error { + var ss serializedFileSet + if err := ss.Read(r); err != nil { + return err + } + + s.mutex.Lock() + s.base = ss.Base + files := make([]*File, len(ss.Files)) + for i := 0; i < len(ss.Files); i++ { + f := &ss.Files[i] + files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos} + } + s.files = files + s.last = nil + s.mutex.Unlock() + + return nil +} + +// Write writes the fileset s to w. +func (s *FileSet) Write(w io.Writer) os.Error { + var ss serializedFileSet + + s.mutex.Lock() + ss.Base = s.base + files := make([]serializedFile, len(s.files)) + for i, f := range s.files { + files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos} + } + ss.Files = files + s.mutex.Unlock() + + return ss.Write(w) +} diff --git a/libgo/go/go/token/serialize_test.go b/libgo/go/go/token/serialize_test.go new file mode 100644 index 0000000..24e419ab --- /dev/null +++ b/libgo/go/go/token/serialize_test.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "bytes" + "fmt" + "os" + "testing" +) + +// equal returns nil if p and q describe the same file set; +// otherwise it returns an error describing the discrepancy. +func equal(p, q *FileSet) os.Error { + if p == q { + // avoid deadlock if p == q + return nil + } + + // not strictly needed for the test + p.mutex.Lock() + q.mutex.Lock() + defer q.mutex.Unlock() + defer p.mutex.Unlock() + + if p.base != q.base { + return fmt.Errorf("different bases: %d != %d", p.base, q.base) + } + + if len(p.files) != len(q.files) { + return fmt.Errorf("different number of files: %d != %d", len(p.files), len(q.files)) + } + + for i, f := range p.files { + g := q.files[i] + if f.set != p { + return fmt.Errorf("wrong fileset for %q", f.name) + } + if g.set != q { + return fmt.Errorf("wrong fileset for %q", g.name) + } + if f.name != g.name { + return fmt.Errorf("different filenames: %q != %q", f.name, g.name) + } + if f.base != g.base { + return fmt.Errorf("different base for %q: %d != %d", f.name, f.base, g.base) + } + if f.size != g.size { + return fmt.Errorf("different size for %q: %d != %d", f.name, f.size, g.size) + } + for j, l := range f.lines { + m := g.lines[j] + if l != m { + return fmt.Errorf("different offsets for %q", f.name) + } + } + for j, l := range f.infos { + m := g.infos[j] + if l.Offset != m.Offset || l.Filename != m.Filename || l.Line != m.Line { + return fmt.Errorf("different infos for %q", f.name) + } + } + } + + // we don't care about .last - it's just a cache + return nil +} + +func checkSerialize(t *testing.T, p *FileSet) { + var buf bytes.Buffer + if err := p.Write(&buf); err != nil { + t.Errorf("writing fileset failed: %s", err) + return + } + q := NewFileSet() + if err := q.Read(&buf); err != nil { + t.Errorf("reading fileset failed: %s", err) + return + } + if err := equal(p, q); err != nil { + t.Errorf("filesets not identical: %s", err) + } +} + +func TestSerialization(t *testing.T) { + p := NewFileSet() + checkSerialize(t, p) + // add some files + for i := 0; i < 10; i++ { + f := p.AddFile(fmt.Sprintf("file%d", i), p.Base()+i, i*100) + checkSerialize(t, p) + // add some lines and alternative file infos + line := 1000 + for offs := 0; offs < f.Size(); offs += 40 + i { + f.AddLine(offs) + if offs%7 == 0 { + f.AddLineInfo(offs, fmt.Sprintf("file%d", offs), line) + line += 33 + } + } + checkSerialize(t, p) + } +} diff --git a/libgo/go/go/typechecker/scope.go b/libgo/go/go/typechecker/scope.go deleted file mode 100644 index d73d1a4..0000000 --- a/libgo/go/go/typechecker/scope.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// DEPRECATED FILE - WILL GO AWAY EVENTUALLY. -// -// Scope handling is now done in go/parser. -// The functionality here is only present to -// keep the typechecker running for now. - -package typechecker - -import "go/ast" - -func (tc *typechecker) openScope() *ast.Scope { - tc.topScope = ast.NewScope(tc.topScope) - return tc.topScope -} - -func (tc *typechecker) closeScope() { - tc.topScope = tc.topScope.Outer -} - -// declInScope declares an object of a given kind and name in scope and sets the object's Decl and N fields. -// It returns the newly allocated object. If an object with the same name already exists in scope, an error -// is reported and the object is not inserted. -func (tc *typechecker) declInScope(scope *ast.Scope, kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object { - obj := ast.NewObj(kind, name.Name) - obj.Decl = decl - //obj.N = n - name.Obj = obj - if name.Name != "_" { - if alt := scope.Insert(obj); alt != nil { - tc.Errorf(name.Pos(), "%s already declared at %s", name.Name, tc.fset.Position(alt.Pos()).String()) - } - } - return obj -} - -// decl is the same as declInScope(tc.topScope, ...) -func (tc *typechecker) decl(kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object { - return tc.declInScope(tc.topScope, kind, name, decl, n) -} - -// find returns the object with the given name if visible in the current scope hierarchy. -// If no such object is found, an error is reported and a bad object is returned instead. -func (tc *typechecker) find(name *ast.Ident) (obj *ast.Object) { - for s := tc.topScope; s != nil && obj == nil; s = s.Outer { - obj = s.Lookup(name.Name) - } - if obj == nil { - tc.Errorf(name.Pos(), "%s not declared", name.Name) - obj = ast.NewObj(ast.Bad, name.Name) - } - name.Obj = obj - return -} - -// findField returns the object with the given name if visible in the type's scope. -// If no such object is found, an error is reported and a bad object is returned instead. -func (tc *typechecker) findField(typ *Type, name *ast.Ident) (obj *ast.Object) { - // TODO(gri) This is simplistic at the moment and ignores anonymous fields. - obj = typ.Scope.Lookup(name.Name) - if obj == nil { - tc.Errorf(name.Pos(), "%s not declared", name.Name) - obj = ast.NewObj(ast.Bad, name.Name) - } - return -} diff --git a/libgo/go/go/typechecker/testdata/test0.src b/libgo/go/go/typechecker/testdata/test0.src deleted file mode 100644 index 4e317f2..0000000 --- a/libgo/go/go/typechecker/testdata/test0.src +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// type declarations - -package P0 - -type ( - B bool - I int32 - A [10]P - T struct { - x, y P - } - P *T - R *R - F func(A) I - Y interface { - f(A) I - } - S []P - M map[I]F - C chan<- I -) - -type ( - a/* ERROR "illegal cycle" */ a - a/* ERROR "already declared" */ int - - b/* ERROR "illegal cycle" */ c - c d - d e - e b /* ERROR "not a type" */ - - t *t - - U V - V W - W *U - - P1 *S2 - P2 P1 - - S1 struct { - a, b, c int - u, v, a/* ERROR "already declared" */ float - } - S2/* ERROR "illegal cycle" */ struct { - x S2 - } - - L1 []L1 - L2 []int - - A1 [10]int - A2/* ERROR "illegal cycle" */ [10]A2 - A3/* ERROR "illegal cycle" */ [10]struct { - x A4 - } - A4 [10]A3 - - F1 func() - F2 func(x, y, z float) - F3 func(x, y, x /* ERROR "already declared" */ float) - F4 func() (x, y, x /* ERROR "already declared" */ float) - F5 func(x int) (x /* ERROR "already declared" */ float) - - I1 interface{} - I2 interface { - m1() - } - I3 interface { - m1() - m1 /* ERROR "already declared" */ () - } - I4 interface { - m1(x, y, x /* ERROR "already declared" */ float) - m2() (x, y, x /* ERROR "already declared" */ float) - m3(x int) (x /* ERROR "already declared" */ float) - } - I5 interface { - m1(I5) - } - - C1 chan int - C2 <-chan int - C3 chan<- C3 - - M1 map[Last]string - M2 map[string]M2 - - Last int -) diff --git a/libgo/go/go/typechecker/testdata/test1.src b/libgo/go/go/typechecker/testdata/test1.src deleted file mode 100644 index b5531fb..0000000 --- a/libgo/go/go/typechecker/testdata/test1.src +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// const and var declarations - -package P1 - -const ( - c1 = 0 - c2 int = 0 - c3, c4 = 0 -) diff --git a/libgo/go/go/typechecker/testdata/test3.src b/libgo/go/go/typechecker/testdata/test3.src deleted file mode 100644 index 2e1a9fa..0000000 --- a/libgo/go/go/typechecker/testdata/test3.src +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package P3 - -// function and method signatures - -func _() {} -func _() {} -func _(x, x /* ERROR "already declared" */ int) {} - -func f() {} -func f /* ERROR "already declared" */ () {} - -func (*foo /* ERROR "invalid receiver" */ ) m() {} -func (bar /* ERROR "not a type" */ ) m() {} - -func f1(x, _, _ int) (_, _ float) {} -func f2(x, y, x /* ERROR "already declared" */ int) {} -func f3(x, y int) (a, b, x /* ERROR "already declared" */ int) {} - -func (x *T) m1() {} -func (x *T) m1 /* ERROR "already declared" */ () {} -func (x T) m1 /* ERROR "already declared" */ () {} -func (T) m1 /* ERROR "already declared" */ () {} - -func (x *T) m2(u, x /* ERROR "already declared" */ int) {} -func (x *T) m3(a, b, c int) (u, x /* ERROR "already declared" */ int) {} -// The following are disabled for now because the typechecker -// in in the process of being rewritten and cannot handle them -// at the moment -//func (T) _(x, x /* "already declared" */ int) {} -//func (T) _() (x, x /* "already declared" */ int) {} - -//func (PT) _() {} - -var bar int - -type T struct{} -type PT (T) diff --git a/libgo/go/go/typechecker/testdata/test4.src b/libgo/go/go/typechecker/testdata/test4.src deleted file mode 100644 index 94d3558..0000000 --- a/libgo/go/go/typechecker/testdata/test4.src +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Constant declarations - -package P4 - -const ( - c0 = 0 -) diff --git a/libgo/go/go/typechecker/type.go b/libgo/go/go/typechecker/type.go deleted file mode 100644 index 1b88eb5..0000000 --- a/libgo/go/go/typechecker/type.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package typechecker - -import "go/ast" - -// A Type represents a Go type. -type Type struct { - Form Form - Obj *ast.Object // corresponding type name, or nil - Scope *ast.Scope // fields and methods, always present - N uint // basic type id, array length, number of function results, or channel direction - Key, Elt *Type // map key and array, pointer, slice, map or channel element - Params *ast.Scope // function (receiver, input and result) parameters, tuple expressions (results of function calls), or nil - Expr ast.Expr // corresponding AST expression -} - -// NewType creates a new type of a given form. -func NewType(form Form) *Type { - return &Type{Form: form, Scope: ast.NewScope(nil)} -} - -// Form describes the form of a type. -type Form int - -// The list of possible type forms. -const ( - BadType Form = iota // for error handling - Unresolved // type not fully setup - Basic - Array - Struct - Pointer - Function - Method - Interface - Slice - Map - Channel - Tuple -) - -var formStrings = [...]string{ - BadType: "badType", - Unresolved: "unresolved", - Basic: "basic", - Array: "array", - Struct: "struct", - Pointer: "pointer", - Function: "function", - Method: "method", - Interface: "interface", - Slice: "slice", - Map: "map", - Channel: "channel", - Tuple: "tuple", -} - -func (form Form) String() string { return formStrings[form] } - -// The list of basic type id's. -const ( - Bool = iota - Byte - Uint - Int - Float - Complex - Uintptr - String - - Uint8 - Uint16 - Uint32 - Uint64 - - Int8 - Int16 - Int32 - Int64 - - Float32 - Float64 - - Complex64 - Complex128 - - // TODO(gri) ideal types are missing -) - -var BasicTypes = map[uint]string{ - Bool: "bool", - Byte: "byte", - Uint: "uint", - Int: "int", - Float: "float", - Complex: "complex", - Uintptr: "uintptr", - String: "string", - - Uint8: "uint8", - Uint16: "uint16", - Uint32: "uint32", - Uint64: "uint64", - - Int8: "int8", - Int16: "int16", - Int32: "int32", - Int64: "int64", - - Float32: "float32", - Float64: "float64", - - Complex64: "complex64", - Complex128: "complex128", -} diff --git a/libgo/go/go/typechecker/typechecker.go b/libgo/go/go/typechecker/typechecker.go deleted file mode 100644 index 2448016..0000000 --- a/libgo/go/go/typechecker/typechecker.go +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// DEPRECATED PACKAGE - SEE go/types INSTEAD. -// This package implements typechecking of a Go AST. -// The result of the typecheck is an augmented AST -// with object and type information for each identifier. -// -package typechecker - -import ( - "fmt" - "go/ast" - "go/token" - "go/scanner" - "os" -) - -// TODO(gri) don't report errors for objects/types that are marked as bad. - - -const debug = true // set for debugging output - -// An importer takes an import path and returns the data describing the -// respective package's exported interface. The data format is TBD. -// -type Importer func(path string) ([]byte, os.Error) - -// CheckPackage typechecks a package and augments the AST by setting -// *ast.Object, *ast.Type, and *ast.Scope fields accordingly. If an -// importer is provided, it is used to handle imports, otherwise they -// are ignored (likely leading to typechecking errors). -// -// If errors are reported, the AST may be incompletely augmented (fields -// may be nil) or contain incomplete object, type, or scope information. -// -func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error { - var tc typechecker - tc.fset = fset - tc.importer = importer - tc.checkPackage(pkg) - return tc.GetError(scanner.Sorted) -} - -// CheckFile typechecks a single file, but otherwise behaves like -// CheckPackage. If the complete package consists of more than just -// one file, the file may not typecheck without errors. -// -func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error { - // create a single-file dummy package - pkg := &ast.Package{file.Name.Name, nil, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}} - return CheckPackage(fset, pkg, importer) -} - -// ---------------------------------------------------------------------------- -// Typechecker state - -type typechecker struct { - fset *token.FileSet - scanner.ErrorVector - importer Importer - globals []*ast.Object // list of global objects - topScope *ast.Scope // current top-most scope - cyclemap map[*ast.Object]bool // for cycle detection - iota int // current value of iota -} - -func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) { - tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...)) -} - -func assert(pred bool) { - if !pred { - panic("internal error") - } -} - -/* -Typechecking is done in several phases: - -phase 1: declare all global objects; also collect all function and method declarations - - all objects have kind, name, decl fields; the decl field permits - quick lookup of an object's declaration - - constant objects have an iota value - - type objects have unresolved types with empty scopes, all others have nil types - - report global double declarations - -phase 2: bind methods to their receiver base types - - receiver base types must be declared in the package, thus for - each method a corresponding (unresolved) type must exist - - report method double declarations and errors with base types - -phase 3: resolve all global objects - - sequentially iterate through all objects in the global scope - - resolve types for all unresolved types and assign types to - all attached methods - - assign types to all other objects, possibly by evaluating - constant and initializer expressions - - resolution may recurse; a cyclemap is used to detect cycles - - report global typing errors - -phase 4: sequentially typecheck function and method bodies - - all global objects are declared and have types and values; - all methods have types - - sequentially process statements in each body; any object - referred to must be fully defined at this point - - report local typing errors -*/ - -func (tc *typechecker) checkPackage(pkg *ast.Package) { - // setup package scope - tc.topScope = Universe - tc.openScope() - defer tc.closeScope() - - // TODO(gri) there's no file scope at the moment since we ignore imports - - // phase 1: declare all global objects; also collect all function and method declarations - var funcs []*ast.FuncDecl - for _, file := range pkg.Files { - for _, decl := range file.Decls { - tc.declGlobal(decl) - if f, isFunc := decl.(*ast.FuncDecl); isFunc { - funcs = append(funcs, f) - } - } - } - - // phase 2: bind methods to their receiver base types - for _, m := range funcs { - if m.Recv != nil { - tc.bindMethod(m) - } - } - - // phase 3: resolve all global objects - tc.cyclemap = make(map[*ast.Object]bool) - for _, obj := range tc.globals { - tc.resolve(obj) - } - assert(len(tc.cyclemap) == 0) - - // 4: sequentially typecheck function and method bodies - for _, f := range funcs { - ftype, _ := f.Name.Obj.Type.(*Type) - tc.checkBlock(f.Body.List, ftype) - } - - pkg.Scope = tc.topScope -} - -func (tc *typechecker) declGlobal(global ast.Decl) { - switch d := global.(type) { - case *ast.BadDecl: - // ignore - - case *ast.GenDecl: - iota := 0 - var prev *ast.ValueSpec - for _, spec := range d.Specs { - switch s := spec.(type) { - case *ast.ImportSpec: - // TODO(gri) imports go into file scope - case *ast.ValueSpec: - switch d.Tok { - case token.CONST: - if s.Values == nil { - // create a new spec with type and values from the previous one - if prev != nil { - s = &ast.ValueSpec{s.Doc, s.Names, prev.Type, prev.Values, s.Comment} - } else { - // TODO(gri) this should probably go into the const decl code - tc.Errorf(s.Pos(), "missing initializer for const %s", s.Names[0].Name) - } - } - for _, name := range s.Names { - tc.globals = append(tc.globals, tc.decl(ast.Con, name, s, iota)) - } - case token.VAR: - for _, name := range s.Names { - tc.globals = append(tc.globals, tc.decl(ast.Var, name, s, 0)) - } - default: - panic("unreachable") - } - prev = s - iota++ - case *ast.TypeSpec: - obj := tc.decl(ast.Typ, s.Name, s, 0) - tc.globals = append(tc.globals, obj) - // give all type objects an unresolved type so - // that we can collect methods in the type scope - typ := NewType(Unresolved) - obj.Type = typ - typ.Obj = obj - default: - panic("unreachable") - } - } - - case *ast.FuncDecl: - if d.Recv == nil { - tc.globals = append(tc.globals, tc.decl(ast.Fun, d.Name, d, 0)) - } - - default: - panic("unreachable") - } -} - -// If x is of the form *T, deref returns T, otherwise it returns x. -func deref(x ast.Expr) ast.Expr { - if p, isPtr := x.(*ast.StarExpr); isPtr { - x = p.X - } - return x -} - -func (tc *typechecker) bindMethod(method *ast.FuncDecl) { - // a method is declared in the receiver base type's scope - var scope *ast.Scope - base := deref(method.Recv.List[0].Type) - if name, isIdent := base.(*ast.Ident); isIdent { - // if base is not an *ast.Ident, we had a syntax - // error and the parser reported an error already - obj := tc.topScope.Lookup(name.Name) - if obj == nil { - tc.Errorf(name.Pos(), "invalid receiver: %s is not declared in this package", name.Name) - } else if obj.Kind != ast.Typ { - tc.Errorf(name.Pos(), "invalid receiver: %s is not a type", name.Name) - } else { - typ := obj.Type.(*Type) - assert(typ.Form == Unresolved) - scope = typ.Scope - } - } - if scope == nil { - // no receiver type found; use a dummy scope - // (we still want to type-check the method - // body, so make sure there is a name object - // and type) - // TODO(gri) should we record the scope so - // that we don't lose the receiver for type- - // checking of the method body? - scope = ast.NewScope(nil) - } - tc.declInScope(scope, ast.Fun, method.Name, method, 0) -} - -func (tc *typechecker) resolve(obj *ast.Object) { - // check for declaration cycles - if tc.cyclemap[obj] { - tc.Errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name) - obj.Kind = ast.Bad - return - } - tc.cyclemap[obj] = true - defer func() { - tc.cyclemap[obj] = false, false - }() - - // resolve non-type objects - typ, _ := obj.Type.(*Type) - if typ == nil { - switch obj.Kind { - case ast.Bad: - // ignore - - case ast.Con: - tc.declConst(obj) - - case ast.Var: - tc.declVar(obj) - obj.Type = tc.typeFor(nil, obj.Decl.(*ast.ValueSpec).Type, false) - - case ast.Fun: - obj.Type = NewType(Function) - t := obj.Decl.(*ast.FuncDecl).Type - tc.declSignature(obj.Type.(*Type), nil, t.Params, t.Results) - - default: - // type objects have non-nil types when resolve is called - if debug { - fmt.Printf("kind = %s\n", obj.Kind) - } - panic("unreachable") - } - return - } - - // resolve type objects - if typ.Form == Unresolved { - tc.typeFor(typ, typ.Obj.Decl.(*ast.TypeSpec).Type, false) - - // provide types for all methods - for _, obj := range typ.Scope.Objects { - if obj.Kind == ast.Fun { - assert(obj.Type == nil) - obj.Type = NewType(Method) - f := obj.Decl.(*ast.FuncDecl) - t := f.Type - tc.declSignature(obj.Type.(*Type), f.Recv, t.Params, t.Results) - } - } - } -} - -func (tc *typechecker) checkBlock(body []ast.Stmt, ftype *Type) { - tc.openScope() - defer tc.closeScope() - - // inject function/method parameters into block scope, if any - if ftype != nil { - for _, par := range ftype.Params.Objects { - if par.Name != "_" { - alt := tc.topScope.Insert(par) - assert(alt == nil) // ftype has no double declarations - } - } - } - - for _, stmt := range body { - tc.checkStmt(stmt) - } -} - -// ---------------------------------------------------------------------------- -// Types - -// unparen removes parentheses around x, if any. -func unparen(x ast.Expr) ast.Expr { - if ux, hasParens := x.(*ast.ParenExpr); hasParens { - return unparen(ux.X) - } - return x -} - -func (tc *typechecker) declFields(scope *ast.Scope, fields *ast.FieldList, ref bool) (n uint) { - if fields != nil { - for _, f := range fields.List { - typ := tc.typeFor(nil, f.Type, ref) - for _, name := range f.Names { - fld := tc.declInScope(scope, ast.Var, name, f, 0) - fld.Type = typ - n++ - } - } - } - return n -} - -func (tc *typechecker) declSignature(typ *Type, recv, params, results *ast.FieldList) { - assert((typ.Form == Method) == (recv != nil)) - typ.Params = ast.NewScope(nil) - tc.declFields(typ.Params, recv, true) - tc.declFields(typ.Params, params, true) - typ.N = tc.declFields(typ.Params, results, true) -} - -func (tc *typechecker) typeFor(def *Type, x ast.Expr, ref bool) (typ *Type) { - x = unparen(x) - - // type name - if t, isIdent := x.(*ast.Ident); isIdent { - obj := tc.find(t) - - if obj.Kind != ast.Typ { - tc.Errorf(t.Pos(), "%s is not a type", t.Name) - if def == nil { - typ = NewType(BadType) - } else { - typ = def - typ.Form = BadType - } - typ.Expr = x - return - } - - if !ref { - tc.resolve(obj) // check for cycles even if type resolved - } - typ = obj.Type.(*Type) - - if def != nil { - // new type declaration: copy type structure - def.Form = typ.Form - def.N = typ.N - def.Key, def.Elt = typ.Key, typ.Elt - def.Params = typ.Params - def.Expr = x - typ = def - } - return - } - - // type literal - typ = def - if typ == nil { - typ = NewType(BadType) - } - typ.Expr = x - - switch t := x.(type) { - case *ast.SelectorExpr: - if debug { - fmt.Println("qualified identifier unimplemented") - } - typ.Form = BadType - - case *ast.StarExpr: - typ.Form = Pointer - typ.Elt = tc.typeFor(nil, t.X, true) - - case *ast.ArrayType: - if t.Len != nil { - typ.Form = Array - // TODO(gri) compute the real length - // (this may call resolve recursively) - (*typ).N = 42 - } else { - typ.Form = Slice - } - typ.Elt = tc.typeFor(nil, t.Elt, t.Len == nil) - - case *ast.StructType: - typ.Form = Struct - tc.declFields(typ.Scope, t.Fields, false) - - case *ast.FuncType: - typ.Form = Function - tc.declSignature(typ, nil, t.Params, t.Results) - - case *ast.InterfaceType: - typ.Form = Interface - tc.declFields(typ.Scope, t.Methods, true) - - case *ast.MapType: - typ.Form = Map - typ.Key = tc.typeFor(nil, t.Key, true) - typ.Elt = tc.typeFor(nil, t.Value, true) - - case *ast.ChanType: - typ.Form = Channel - typ.N = uint(t.Dir) - typ.Elt = tc.typeFor(nil, t.Value, true) - - default: - if debug { - fmt.Printf("x is %T\n", x) - } - panic("unreachable") - } - - return -} - -// ---------------------------------------------------------------------------- -// TODO(gri) implement these place holders - -func (tc *typechecker) declConst(*ast.Object) { -} - -func (tc *typechecker) declVar(*ast.Object) { -} - -func (tc *typechecker) checkStmt(ast.Stmt) { -} diff --git a/libgo/go/go/typechecker/typechecker_test.go b/libgo/go/go/typechecker/typechecker_test.go deleted file mode 100644 index 4bad449..0000000 --- a/libgo/go/go/typechecker/typechecker_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements a simple typechecker test harness. Packages found -// in the testDir directory are typechecked. Error messages reported by -// the typechecker are compared against the error messages expected for -// the test files. -// -// Expected errors are indicated in the test files by putting a comment -// of the form /* ERROR "rx" */ immediately following an offending token. -// The harness will verify that an error matching the regular expression -// rx is reported at that source position. Consecutive comments may be -// used to indicate multiple errors for the same token position. -// -// For instance, the following test file indicates that a "not declared" -// error should be reported for the undeclared variable x: -// -// package P0 -// func f() { -// _ = x /* ERROR "not declared" */ + 1 -// } -// -// If the -pkg flag is set, only packages with package names matching -// the regular expression provided via the flag value are tested. - -package typechecker - -import ( - "flag" - "fmt" - "go/ast" - "go/parser" - "go/scanner" - "go/token" - "io/ioutil" - "os" - "regexp" - "sort" - "strings" - "testing" -) - -const testDir = "./testdata" // location of test packages - -var fset = token.NewFileSet() - -var ( - pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name") - trace = flag.Bool("trace", false, "print package names") -) - -// ERROR comments must be of the form /* ERROR "rx" */ and rx is -// a regular expression that matches the expected error message. -var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`) - -// expectedErrors collects the regular expressions of ERROR comments -// found in the package files of pkg and returns them in sorted order -// (by filename and position). -func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) { - // scan all package files - for filename := range pkg.Files { - src, err := ioutil.ReadFile(filename) - if err != nil { - t.Fatalf("expectedErrors(%s): %v", pkg.Name, err) - } - - var s scanner.Scanner - file := fset.AddFile(filename, fset.Base(), len(src)) - s.Init(file, src, nil, scanner.ScanComments) - var prev token.Pos // position of last non-comment token - loop: - for { - pos, tok, lit := s.Scan() - switch tok { - case token.EOF: - break loop - case token.COMMENT: - s := errRx.FindStringSubmatch(lit) - if len(s) == 2 { - list = append(list, &scanner.Error{fset.Position(prev), string(s[1])}) - } - default: - prev = pos - } - } - } - sort.Sort(list) // multiple files may not be sorted - return -} - -func testFilter(f *os.FileInfo) bool { - return strings.HasSuffix(f.Name, ".src") && f.Name[0] != '.' -} - -func checkError(t *testing.T, expected, found *scanner.Error) { - rx, err := regexp.Compile(expected.Msg) - if err != nil { - t.Errorf("%s: %v", expected.Pos, err) - return - } - - match := rx.MatchString(found.Msg) - - if expected.Pos.Offset != found.Pos.Offset { - if match { - t.Errorf("%s: expected error should have been at %s", expected.Pos, found.Pos) - } else { - t.Errorf("%s: error matching %q expected", expected.Pos, expected.Msg) - return - } - } - - if !match { - t.Errorf("%s: %q does not match %q", expected.Pos, expected.Msg, found.Msg) - } -} - -func TestTypeCheck(t *testing.T) { - flag.Parse() - pkgRx, err := regexp.Compile(*pkgPat) - if err != nil { - t.Fatalf("illegal flag value %q: %s", *pkgPat, err) - } - - pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0) - if err != nil { - scanner.PrintError(os.Stderr, err) - t.Fatalf("packages in %s contain syntax errors", testDir) - } - - for _, pkg := range pkgs { - if !pkgRx.MatchString(pkg.Name) { - continue // only test selected packages - } - - if *trace { - fmt.Println(pkg.Name) - } - - xlist := expectedErrors(t, pkg) - err := CheckPackage(fset, pkg, nil) - if err != nil { - if elist, ok := err.(scanner.ErrorList); ok { - // verify that errors match - for i := 0; i < len(xlist) && i < len(elist); i++ { - checkError(t, xlist[i], elist[i]) - } - // the correct number or errors must have been found - if len(xlist) != len(elist) { - fmt.Fprintf(os.Stderr, "%s\n", pkg.Name) - scanner.PrintError(os.Stderr, elist) - fmt.Fprintln(os.Stderr) - t.Errorf("TypeCheck(%s): %d errors expected but %d reported", pkg.Name, len(xlist), len(elist)) - } - } else { - t.Errorf("TypeCheck(%s): %v", pkg.Name, err) - } - } else if len(xlist) > 0 { - t.Errorf("TypeCheck(%s): %d errors expected but 0 reported", pkg.Name, len(xlist)) - } - } -} diff --git a/libgo/go/go/typechecker/universe.go b/libgo/go/go/typechecker/universe.go deleted file mode 100644 index 81c14a0..0000000 --- a/libgo/go/go/typechecker/universe.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package typechecker - -import "go/ast" - -// TODO(gri) should this be in package ast? - -// The Universe scope contains all predeclared identifiers. -var Universe *ast.Scope - -func def(obj *ast.Object) { - alt := Universe.Insert(obj) - if alt != nil { - panic("object declared twice") - } -} - -func init() { - Universe = ast.NewScope(nil) - - // basic types - for n, name := range BasicTypes { - typ := NewType(Basic) - typ.N = n - obj := ast.NewObj(ast.Typ, name) - obj.Type = typ - typ.Obj = obj - def(obj) - } - - // built-in functions - // TODO(gri) implement this -} diff --git a/libgo/go/go/types/testdata/test0.src b/libgo/go/go/types/testdata/test0.src deleted file mode 100644 index 84a1abe..0000000 --- a/libgo/go/go/types/testdata/test0.src +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// type declarations - -package test0 - -import "unsafe" - -const pi = 3.1415 - -type ( - N undeclared /* ERROR "undeclared" */ - B bool - I int32 - A [10]P - T struct { - x, y P - } - P *T - R (*R) - F func(A) I - Y interface { - f(A) I - } - S [](((P))) - M map[I]F - C chan<- I -) - - -type ( - p1 pi /* ERROR "not a package" */ .foo - p2 unsafe.Pointer -) - - -type ( - Pi pi /* ERROR "not a type" */ - - a /* DISABLED "illegal cycle" */ a - a /* ERROR "redeclared" */ int - - // where the cycle error appears depends on the - // order in which declarations are processed - // (which depends on the order in which a map - // is iterated through) - b c - c /* DISABLED "illegal cycle" */ d - d e - e b - - t *t - - U V - V *W - W U - - P1 *S2 - P2 P1 - - S0 struct { - } - S1 struct { - a, b, c int - u, v, a /* ERROR "redeclared" */ float32 - } - S2 struct { - U // anonymous field - // TODO(gri) recognize double-declaration below - // U /* ERROR "redeclared" */ int - } - S3 struct { - x S2 - } - S4/* DISABLED "illegal cycle" */ struct { - S4 - } - S5 struct { - S6 - } - S6 /* DISABLED "illegal cycle" */ struct { - field S7 - } - S7 struct { - S5 - } - - L1 []L1 - L2 []int - - A1 [10]int - A2 /* DISABLED "illegal cycle" */ [10]A2 - A3 /* DISABLED "illegal cycle" */ [10]struct { - x A4 - } - A4 [10]A3 - - F1 func() - F2 func(x, y, z float32) - F3 func(x, y, x /* ERROR "redeclared" */ float32) - F4 func() (x, y, x /* ERROR "redeclared" */ float32) - F5 func(x int) (x /* ERROR "redeclared" */ float32) - F6 func(x ...int) - - I1 interface{} - I2 interface { - m1() - } - I3 interface { - m1() - m1 /* ERROR "redeclared" */ () - } - I4 interface { - m1(x, y, x /* ERROR "redeclared" */ float32) - m2() (x, y, x /* ERROR "redeclared" */ float32) - m3(x int) (x /* ERROR "redeclared" */ float32) - } - I5 interface { - m1(I5) - } - I6 interface { - S0 /* ERROR "non-interface" */ - } - I7 interface { - I1 - I1 - } - I8 /* DISABLED "illegal cycle" */ interface { - I8 - } - I9 /* DISABLED "illegal cycle" */ interface { - I10 - } - I10 interface { - I11 - } - I11 interface { - I9 - } - - C1 chan int - C2 <-chan int - C3 chan<- C3 - C4 chan C5 - C5 chan C6 - C6 chan C4 - - M1 map[Last]string - M2 map[string]M2 - - Last int -) diff --git a/libgo/go/gob/codec_test.go b/libgo/go/gob/codec_test.go index a5fb91c..2bcbf82 100644 --- a/libgo/go/gob/codec_test.go +++ b/libgo/go/gob/codec_test.go @@ -544,7 +544,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a []byte } - instr := &decInstr{decUint8Array, 6, 0, 0, ovfl} + instr := &decInstr{decUint8Slice, 6, 0, 0, ovfl} state := newDecodeStateFromData(bytesResult) execDec("bytes", instr, state, t, unsafe.Pointer(&data)) if string(data.a) != "hello" { diff --git a/libgo/go/gob/decode.go b/libgo/go/gob/decode.go index bf7cb95..f480087 100644 --- a/libgo/go/gob/decode.go +++ b/libgo/go/gob/decode.go @@ -70,13 +70,12 @@ func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Erro if b <= 0x7f { return uint64(b), width, nil } - nb := -int(int8(b)) - if nb > uint64Size { + n := -int(int8(b)) + if n > uint64Size { err = errBadUint return } - var n int - n, err = io.ReadFull(r, buf[0:nb]) + width, err = io.ReadFull(r, buf[0:n]) if err != nil { if err == os.EOF { err = io.ErrUnexpectedEOF @@ -84,11 +83,10 @@ func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Erro return } // Could check that the high byte is zero but it's not worth it. - for i := 0; i < n; i++ { - x <<= 8 - x |= uint64(buf[i]) - width++ + for _, b := range buf[0:width] { + x = x<<8 | uint64(b) } + width++ // +1 for length byte return } @@ -102,19 +100,18 @@ func (state *decoderState) decodeUint() (x uint64) { if b <= 0x7f { return uint64(b) } - nb := -int(int8(b)) - if nb > uint64Size { + n := -int(int8(b)) + if n > uint64Size { error(errBadUint) } - n, err := state.b.Read(state.buf[0:nb]) + width, err := state.b.Read(state.buf[0:n]) if err != nil { error(err) } // Don't need to check error; it's safe to loop regardless. // Could check that the high byte is zero but it's not worth it. - for i := 0; i < n; i++ { - x <<= 8 - x |= uint64(state.buf[i]) + for _, b := range state.buf[0:width] { + x = x<<8 | uint64(b) } return x } @@ -385,19 +382,29 @@ func decComplex128(i *decInstr, state *decoderState, p unsafe.Pointer) { *(*complex128)(p) = complex(real, imag) } -// decUint8Array decodes byte array and stores through p a slice header +// decUint8Slice decodes a byte slice and stores through p a slice header // describing the data. -// uint8 arrays are encoded as an unsigned count followed by the raw bytes. -func decUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) { +// uint8 slices are encoded as an unsigned count followed by the raw bytes. +func decUint8Slice(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new([]uint8)) } p = *(*unsafe.Pointer)(p) } - b := make([]uint8, state.decodeUint()) - state.b.Read(b) - *(*[]uint8)(p) = b + n := int(state.decodeUint()) + if n < 0 { + errorf("negative length decoding []byte") + } + slice := (*[]uint8)(p) + if cap(*slice) < n { + *slice = make([]uint8, n) + } else { + *slice = (*slice)[0:n] + } + if _, err := state.b.Read(*slice); err != nil { + errorf("error decoding []byte: %s", err) + } } // decString decodes byte array and stores through p a string header @@ -457,20 +464,17 @@ func allocate(rtyp reflect.Type, p uintptr, indir int) uintptr { // decodeSingle decodes a top-level value that is not a struct and stores it through p. // Such values are preceded by a zero, making them have the memory layout of a // struct field (although with an illegal field number). -func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, p uintptr) (err os.Error) { - indir := ut.indir - if ut.isGobDecoder { - indir = int(ut.decIndir) - } - p = allocate(ut.base, p, indir) +func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, basep uintptr) (err os.Error) { state := dec.newDecoderState(&dec.buf) state.fieldnum = singletonField - basep := p delta := int(state.decodeUint()) if delta != 0 { errorf("decode: corrupted data: non-zero delta for singleton") } instr := &engine.instr[singletonField] + if instr.indir != ut.indir { + return os.NewError("gob: internal error: inconsistent indirection") + } ptr := unsafe.Pointer(basep) // offset will be zero if instr.indir > 1 { ptr = decIndirect(ptr, instr.indir) @@ -653,12 +657,15 @@ func (dec *Decoder) decodeSlice(atyp reflect.Type, state *decoderState, p uintpt } p = *(*uintptr)(up) } - // Allocate storage for the slice elements, that is, the underlying array. + // Allocate storage for the slice elements, that is, the underlying array, + // if the existing slice does not have the capacity. // Always write a header at p. hdrp := (*reflect.SliceHeader)(unsafe.Pointer(p)) - hdrp.Data = uintptr(unsafe.NewArray(atyp.Elem(), n)) + if hdrp.Cap < n { + hdrp.Data = uintptr(unsafe.NewArray(atyp.Elem(), n)) + hdrp.Cap = n + } hdrp.Len = n - hdrp.Cap = n dec.decodeArrayHelper(state, hdrp.Data, elemOp, elemWid, n, elemIndir, ovfl) } @@ -842,7 +849,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg case reflect.Slice: name = "element of " + name if t.Elem().Kind() == reflect.Uint8 { - op = decUint8Array + op = decUint8Slice break } var elemId typeId @@ -1056,10 +1063,7 @@ func (dec *Decoder) typeString(remoteId typeId) string { // compileSingle compiles the decoder engine for a non-struct top-level value, including // GobDecoders. func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) { - rt := ut.base - if ut.isGobDecoder { - rt = ut.user - } + rt := ut.user engine = new(decEngine) engine.instr = make([]decInstr, 1) // one item name := rt.String() // best we can do @@ -1150,7 +1154,7 @@ func (dec *Decoder) getDecEnginePtr(remoteId typeId, ut *userTypeInfo) (enginePt decoderMap[remoteId] = enginePtr *enginePtr, err = dec.compileDec(remoteId, ut) if err != nil { - decoderMap[remoteId] = nil, false + delete(decoderMap, remoteId) } } return @@ -1175,7 +1179,7 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er *enginePtr, err = dec.compileIgnoreSingle(wireId) } if err != nil { - dec.ignorerCache[wireId] = nil, false + delete(dec.ignorerCache, wireId) } } return @@ -1189,7 +1193,7 @@ func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) { dec.decodeIgnoredValue(wireId) return } - // Dereference down to the underlying struct type. + // Dereference down to the underlying type. ut := userType(val.Type()) base := ut.base var enginePtr **decEngine diff --git a/libgo/go/gob/decoder.go b/libgo/go/gob/decoder.go index 2819471..5efcea8 100644 --- a/libgo/go/gob/decoder.go +++ b/libgo/go/gob/decoder.go @@ -29,9 +29,15 @@ type Decoder struct { } // NewDecoder returns a new decoder that reads from the io.Reader. +// If r does not also implement io.ByteReader, it will be wrapped in a +// bufio.Reader. func NewDecoder(r io.Reader) *Decoder { dec := new(Decoder) - dec.r = bufio.NewReader(r) + // We use the ability to read bytes as a plausible surrogate for buffering. + if _, ok := r.(io.ByteReader); !ok { + r = bufio.NewReader(r) + } + dec.r = r dec.wireType = make(map[typeId]*wireType) dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine) dec.ignorerCache = make(map[typeId]**decEngine) @@ -58,6 +64,8 @@ func (dec *Decoder) recvType(id typeId) { dec.wireType[id] = wire } +var errBadCount = gobError{os.NewError("invalid message length")} + // recvMessage reads the next count-delimited item from the input. It is the converse // of Encoder.writeMessage. It returns false on EOF or other error reading the message. func (dec *Decoder) recvMessage() bool { @@ -67,6 +75,10 @@ func (dec *Decoder) recvMessage() bool { dec.err = err return false } + if nbytes >= 1<<31 { + dec.err = errBadCount + return false + } dec.readMessage(int(nbytes)) return dec.err == nil } diff --git a/libgo/go/gob/doc.go b/libgo/go/gob/doc.go index a9284ce..05ebef1 100644 --- a/libgo/go/gob/doc.go +++ b/libgo/go/gob/doc.go @@ -68,7 +68,10 @@ the destination variable must be able to represent the value or the decode operation will fail. Structs, arrays and slices are also supported. Strings and arrays of bytes are -supported with a special, efficient representation (see below). +supported with a special, efficient representation (see below). When a slice is +decoded, if the existing slice has capacity the slice will be extended in place; +if not, a new array is allocated. Regardless, the length of the resuling slice +reports the number of elements decoded. Functions and channels cannot be sent in a gob. Attempting to encode a value that contains one will fail. diff --git a/libgo/go/gob/encode.go b/libgo/go/gob/encode.go index 317014e..6bb5458 100644 --- a/libgo/go/gob/encode.go +++ b/libgo/go/gob/encode.go @@ -59,15 +59,14 @@ func (state *encoderState) encodeUint(x uint64) { } return } - var n, m int - m = uint64Size - for n = 1; x > 0; n++ { - state.buf[m] = uint8(x) + i := uint64Size + for x > 0 { + state.buf[i] = uint8(x) x >>= 8 - m-- + i-- } - state.buf[m] = uint8(-(n - 1)) - n, err := state.b.Write(state.buf[m : uint64Size+1]) + state.buf[i] = uint8(i - uint64Size) // = loop count, negated + _, err := state.b.Write(state.buf[i : uint64Size+1]) if err != nil { error(err) } @@ -454,6 +453,7 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { // should be written to b, before the encoded value. enc.pushWriter(b) data := new(bytes.Buffer) + data.Write(spaceForLength) enc.encode(data, iv.Elem(), ut) if enc.err != nil { error(enc.err) diff --git a/libgo/go/gob/encoder.go b/libgo/go/gob/encoder.go index 96101d9..878d082 100644 --- a/libgo/go/gob/encoder.go +++ b/libgo/go/gob/encoder.go @@ -20,11 +20,16 @@ type Encoder struct { sent map[reflect.Type]typeId // which types we've already sent countState *encoderState // stage for writing counts freeList *encoderState // list of free encoderStates; avoids reallocation - buf []byte // for collecting the output. byteBuf bytes.Buffer // buffer for top-level encoderState err os.Error } +// Before we encode a message, we reserve space at the head of the +// buffer in which to encode its length. This means we can use the +// buffer to assemble the message without another allocation. +const maxLength = 9 // Maximum size of an encoded length. +var spaceForLength = make([]byte, maxLength) + // NewEncoder returns a new encoder that will transmit on the io.Writer. func NewEncoder(w io.Writer) *Encoder { enc := new(Encoder) @@ -61,20 +66,22 @@ func (enc *Encoder) setError(err os.Error) { // writeMessage sends the data item preceded by a unsigned count of its length. func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) { - enc.countState.encodeUint(uint64(b.Len())) - // Build the buffer. - countLen := enc.countState.b.Len() - total := countLen + b.Len() - if total > len(enc.buf) { - enc.buf = make([]byte, total+1000) // extra for growth - } - // Place the length before the data. - // TODO(r): avoid the extra copy here. - enc.countState.b.Read(enc.buf[0:countLen]) - // Now the data. - b.Read(enc.buf[countLen:total]) + // Space has been reserved for the length at the head of the message. + // This is a little dirty: we grab the slice from the bytes.Buffer and massage + // it by hand. + message := b.Bytes() + messageLen := len(message) - maxLength + // Encode the length. + enc.countState.b.Reset() + enc.countState.encodeUint(uint64(messageLen)) + // Copy the length to be a prefix of the message. + offset := maxLength - enc.countState.b.Len() + copy(message[offset:], enc.countState.b.Bytes()) // Write the data. - _, err := w.Write(enc.buf[0:total]) + _, err := w.Write(message[offset:]) + // Drain the buffer and restore the space at the front for the count of the next message. + b.Reset() + b.Write(spaceForLength) if err != nil { enc.setError(err) } @@ -224,6 +231,7 @@ func (enc *Encoder) EncodeValue(value reflect.Value) os.Error { enc.err = nil enc.byteBuf.Reset() + enc.byteBuf.Write(spaceForLength) state := enc.newEncoderState(&enc.byteBuf) enc.sendTypeDescriptor(enc.writer(), state, ut) diff --git a/libgo/go/gob/encoder_test.go b/libgo/go/gob/encoder_test.go index f5ee423..a774438 100644 --- a/libgo/go/gob/encoder_test.go +++ b/libgo/go/gob/encoder_test.go @@ -575,6 +575,91 @@ func TestGobMapInterfaceEncode(t *testing.T) { enc := NewEncoder(buf) err := enc.Encode(m) if err != nil { - t.Errorf("gob.Encode map: %s", err) + t.Errorf("encode map: %s", err) + } +} + +func TestSliceReusesMemory(t *testing.T) { + buf := bytes.NewBuffer(nil) + // Bytes + { + x := []byte("abcd") + enc := NewEncoder(buf) + err := enc.Encode(x) + if err != nil { + t.Errorf("bytes: encode: %s", err) + } + // Decode into y, which is big enough. + y := []byte("ABCDE") + addr := &y[0] + dec := NewDecoder(buf) + err = dec.Decode(&y) + if err != nil { + t.Fatal("bytes: decode:", err) + } + if !bytes.Equal(x, y) { + t.Errorf("bytes: expected %q got %q\n", x, y) + } + if addr != &y[0] { + t.Errorf("bytes: unnecessary reallocation") + } + } + // general slice + { + x := []int("abcd") + enc := NewEncoder(buf) + err := enc.Encode(x) + if err != nil { + t.Errorf("ints: encode: %s", err) + } + // Decode into y, which is big enough. + y := []int("ABCDE") + addr := &y[0] + dec := NewDecoder(buf) + err = dec.Decode(&y) + if err != nil { + t.Fatal("ints: decode:", err) + } + if !reflect.DeepEqual(x, y) { + t.Errorf("ints: expected %q got %q\n", x, y) + } + if addr != &y[0] { + t.Errorf("ints: unnecessary reallocation") + } + } +} + +// Used to crash: negative count in recvMessage. +func TestBadCount(t *testing.T) { + b := []byte{0xfb, 0xa5, 0x82, 0x2f, 0xca, 0x1} + if err := NewDecoder(bytes.NewBuffer(b)).Decode(nil); err == nil { + t.Error("expected error from bad count") + } else if err.String() != errBadCount.String() { + t.Error("expected bad count error; got", err) + } +} + +// Verify that sequential Decoders built on a single input will +// succeed if the input implements ReadByte and there is no +// type information in the stream. +func TestSequentialDecoder(t *testing.T) { + b := new(bytes.Buffer) + enc := NewEncoder(b) + const count = 10 + for i := 0; i < count; i++ { + s := fmt.Sprintf("%d", i) + if err := enc.Encode(s); err != nil { + t.Error("encoder fail:", err) + } + } + for i := 0; i < count; i++ { + dec := NewDecoder(b) + var s string + if err := dec.Decode(&s); err != nil { + t.Fatal("decoder fail:", err) + } + if s != fmt.Sprintf("%d", i) { + t.Fatalf("decode expected %d got %s", i, s) + } } } diff --git a/libgo/go/gob/gobencdec_test.go b/libgo/go/gob/gobencdec_test.go index 371a43c..01addbe 100644 --- a/libgo/go/gob/gobencdec_test.go +++ b/libgo/go/gob/gobencdec_test.go @@ -424,7 +424,7 @@ func TestGobEncoderNonStructSingleton(t *testing.T) { t.Fatal("decode error:", err) } if x != 1234 { - t.Errorf("expected 1234 got %c", x) + t.Errorf("expected 1234 got %d", x) } } @@ -488,3 +488,40 @@ func TestGobEncoderIgnoreNilEncoder(t *testing.T) { t.Errorf("expected x.G = nil, got %v", x.G) } } + +type gobDecoderBug0 struct { + foo, bar string +} + +func (br *gobDecoderBug0) String() string { + return br.foo + "-" + br.bar +} + +func (br *gobDecoderBug0) GobEncode() ([]byte, os.Error) { + return []byte(br.String()), nil +} + +func (br *gobDecoderBug0) GobDecode(b []byte) os.Error { + br.foo = "foo" + br.bar = "bar" + return nil +} + +// This was a bug: the receiver has a different indirection level +// than the variable. +func TestGobEncoderExtraIndirect(t *testing.T) { + gdb := &gobDecoderBug0{"foo", "bar"} + buf := new(bytes.Buffer) + e := NewEncoder(buf) + if err := e.Encode(gdb); err != nil { + t.Fatalf("encode: %v", err) + } + d := NewDecoder(buf) + var got *gobDecoderBug0 + if err := d.Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.foo != gdb.foo || got.bar != gdb.bar { + t.Errorf("got = %q, want %q", got, gdb) + } +} diff --git a/libgo/go/gob/type.go b/libgo/go/gob/type.go index b2f716c..870101e 100644 --- a/libgo/go/gob/type.go +++ b/libgo/go/gob/type.go @@ -405,7 +405,7 @@ func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os. var type0, type1 gobType defer func() { if err != nil { - types[rt] = nil, false + delete(types, rt) } }() // Install the top-level type before the subtypes (e.g. struct before diff --git a/libgo/go/html/escape.go b/libgo/go/html/escape.go index 0de97c5..e9edc47 100644 --- a/libgo/go/html/escape.go +++ b/libgo/go/html/escape.go @@ -6,6 +6,7 @@ package html import ( "bytes" + "os" "strings" "utf8" ) @@ -182,12 +183,24 @@ func unescape(b []byte) []byte { return b } +// lower lower-cases the A-Z bytes in b in-place, so that "aBc" becomes "abc". +func lower(b []byte) []byte { + for i, c := range b { + if 'A' <= c && c <= 'Z' { + b[i] = c + 'a' - 'A' + } + } + return b +} + const escapedChars = `&'<>"` -func escape(buf *bytes.Buffer, s string) { +func escape(w writer, s string) os.Error { i := strings.IndexAny(s, escapedChars) for i != -1 { - buf.WriteString(s[0:i]) + if _, err := w.WriteString(s[:i]); err != nil { + return err + } var esc string switch s[i] { case '&': @@ -204,10 +217,13 @@ func escape(buf *bytes.Buffer, s string) { panic("unrecognized escape character") } s = s[i+1:] - buf.WriteString(esc) + if _, err := w.WriteString(esc); err != nil { + return err + } i = strings.IndexAny(s, escapedChars) } - buf.WriteString(s) + _, err := w.WriteString(s) + return err } // EscapeString escapes special characters like "<" to become "<". It diff --git a/libgo/go/html/parse.go b/libgo/go/html/parse.go index 519ebe5..582437f 100644 --- a/libgo/go/html/parse.go +++ b/libgo/go/html/parse.go @@ -29,6 +29,9 @@ type parser struct { head, form *Node // Other parsing state flags (section 11.2.3.5). scripting, framesetOK bool + // originalIM is the insertion mode to go back to after completing a text + // or inTableText insertion mode. + originalIM insertionMode } func (p *parser) top() *Node { @@ -64,21 +67,37 @@ var ( // popUntil([]string{"html, "table"}, "table") would return true and leave: // ["html", "body", "font"] func (p *parser) popUntil(stopTags []string, matchTags ...string) bool { + if i := p.indexOfElementInScope(stopTags, matchTags...); i != -1 { + p.oe = p.oe[:i] + return true + } + return false +} + +// indexOfElementInScope returns the index in p.oe of the highest element +// whose tag is in matchTags that is in scope according to stopTags. +// If no matching element is in scope, it returns -1. +func (p *parser) indexOfElementInScope(stopTags []string, matchTags ...string) int { for i := len(p.oe) - 1; i >= 0; i-- { tag := p.oe[i].Data for _, t := range matchTags { if t == tag { - p.oe = p.oe[:i] - return true + return i } } for _, t := range stopTags { if t == tag { - return false + return -1 } } } - return false + return -1 +} + +// elementInScope is like popUntil, except that it doesn't modify the stack of +// open elements. +func (p *parser) elementInScope(stopTags []string, matchTags ...string) bool { + return p.indexOfElementInScope(stopTags, matchTags...) != -1 } // addChild adds a child node n to the top element, and pushes n onto the stack @@ -198,12 +217,23 @@ type insertionMode func(*parser) (insertionMode, bool) // Section 11.2.3.1, "using the rules for". func useTheRulesFor(p *parser, actual, delegate insertionMode) (insertionMode, bool) { im, consumed := delegate(p) + // TODO: do we need to update p.originalMode if it equals delegate? if im != delegate { return im, consumed } return actual, consumed } +// setOriginalIM sets the insertion mode to return to after completing a text or +// inTableText insertion mode. +// Section 11.2.3.1, "using the rules for". +func (p *parser) setOriginalIM(im insertionMode) { + if p.originalIM != nil { + panic("html: bad parser state: originalIM was set twice") + } + p.originalIM = im +} + // Section 11.2.5.4.1. func initialIM(p *parser) (insertionMode, bool) { if p.tok.Type == DoctypeToken { @@ -302,8 +332,10 @@ func inHeadIM(p *parser) (insertionMode, bool) { switch p.tok.Data { case "meta": // TODO. - case "script": - // TODO. + case "script", "title": + p.addElement(p.tok.Data, p.tok.Attr) + p.setOriginalIM(inHeadIM) + return textIM, true default: implied = true } @@ -365,7 +397,6 @@ func afterHeadIM(p *parser) (insertionMode, bool) { // Section 11.2.5.4.7. func inBodyIM(p *parser) (insertionMode, bool) { - var endP bool switch p.tok.Type { case TextToken: p.reconstructActiveFormattingElements() @@ -374,15 +405,10 @@ func inBodyIM(p *parser) (insertionMode, bool) { case StartTagToken: switch p.tok.Data { case "address", "article", "aside", "blockquote", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "menu", "nav", "ol", "p", "section", "summary", "ul": - // TODO: Do the proper "does the stack of open elements has a p element in button scope" algorithm in section 11.2.3.2. - n := p.top() - if n.Type == ElementNode && n.Data == "p" { - endP = true - } else { - p.addElement(p.tok.Data, p.tok.Attr) - } + p.popUntil(buttonScopeStopTags, "p") + p.addElement(p.tok.Data, p.tok.Attr) case "h1", "h2", "h3", "h4", "h5", "h6": - // TODO: auto-insert </p> if necessary. + p.popUntil(buttonScopeStopTags, "p") switch n := p.top(); n.Data { case "h1", "h2", "h3", "h4", "h5", "h6": p.oe.pop() @@ -399,6 +425,11 @@ func inBodyIM(p *parser) (insertionMode, bool) { case "b", "big", "code", "em", "font", "i", "s", "small", "strike", "strong", "tt", "u": p.reconstructActiveFormattingElements() p.addFormattingElement(p.tok.Data, p.tok.Attr) + case "applet", "marquee", "object": + p.reconstructActiveFormattingElements() + p.addElement(p.tok.Data, p.tok.Attr) + p.afe = append(p.afe, &scopeMarker) + p.framesetOK = false case "area", "br", "embed", "img", "input", "keygen", "wbr": p.reconstructActiveFormattingElements() p.addElement(p.tok.Data, p.tok.Attr) @@ -406,12 +437,12 @@ func inBodyIM(p *parser) (insertionMode, bool) { p.acknowledgeSelfClosingTag() p.framesetOK = false case "table": - // TODO: auto-insert </p> if necessary, depending on quirks mode. + p.popUntil(buttonScopeStopTags, "p") // TODO: skip this step in quirks mode. p.addElement(p.tok.Data, p.tok.Attr) p.framesetOK = false return inTableIM, true case "hr": - // TODO: auto-insert </p> if necessary. + p.popUntil(buttonScopeStopTags, "p") p.addElement(p.tok.Data, p.tok.Attr) p.oe.pop() p.acknowledgeSelfClosingTag() @@ -425,6 +456,11 @@ func inBodyIM(p *parser) (insertionMode, bool) { case "body": // TODO: autoclose the stack of open elements. return afterBodyIM, true + case "p": + if !p.elementInScope(buttonScopeStopTags, "p") { + p.addElement("p", nil) + } + p.popUntil(buttonScopeStopTags, "p") case "a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u": p.inBodyEndTagFormatting(p.tok.Data) default: @@ -434,14 +470,8 @@ func inBodyIM(p *parser) (insertionMode, bool) { } } } - if endP { - // TODO: do the proper algorithm. - n := p.oe.pop() - if n.Type != ElementNode || n.Data != "p" { - panic("unreachable") - } - } - return inBodyIM, !endP + + return inBodyIM, true } func (p *parser) inBodyEndTagFormatting(tag string) { @@ -560,6 +590,20 @@ func (p *parser) inBodyEndTagFormatting(tag string) { } } +// Section 11.2.5.4.8. +func textIM(p *parser) (insertionMode, bool) { + switch p.tok.Type { + case TextToken: + p.addText(p.tok.Data) + return textIM, true + case EndTagToken: + p.oe.pop() + } + o := p.originalIM + p.originalIM = nil + return o, p.tok.Type == EndTagToken +} + // Section 11.2.5.4.9. func inTableIM(p *parser) (insertionMode, bool) { var ( diff --git a/libgo/go/html/parse_test.go b/libgo/go/html/parse_test.go index 7d918d2..564580c 100644 --- a/libgo/go/html/parse_test.go +++ b/libgo/go/html/parse_test.go @@ -80,13 +80,13 @@ func dumpLevel(w io.Writer, n *Node, level int) os.Error { case DocumentNode: return os.NewError("unexpected DocumentNode") case ElementNode: - fmt.Fprintf(w, "<%s>", EscapeString(n.Data)) + fmt.Fprintf(w, "<%s>", n.Data) case TextNode: - fmt.Fprintf(w, "%q", EscapeString(n.Data)) + fmt.Fprintf(w, "%q", n.Data) case CommentNode: return os.NewError("COMMENT") case DoctypeNode: - fmt.Fprintf(w, "<!DOCTYPE %s>", EscapeString(n.Data)) + fmt.Fprintf(w, "<!DOCTYPE %s>", n.Data) case scopeMarkerNode: return os.NewError("unexpected scopeMarkerNode") default: @@ -123,7 +123,7 @@ func TestParser(t *testing.T) { rc := make(chan io.Reader) go readDat(filename, rc) // TODO(nigeltao): Process all test cases, not just a subset. - for i := 0; i < 25; i++ { + for i := 0; i < 27; i++ { // Parse the #data section. b, err := ioutil.ReadAll(<-rc) if err != nil { @@ -134,7 +134,7 @@ func TestParser(t *testing.T) { if err != nil { t.Fatal(err) } - actual, err := dump(doc) + got, err := dump(doc) if err != nil { t.Fatal(err) } @@ -147,9 +147,26 @@ func TestParser(t *testing.T) { if err != nil { t.Fatal(err) } - expected := string(b) - if actual != expected { - t.Errorf("%s test #%d %q, actual vs expected:\n----\n%s----\n%s----", filename, i, text, actual, expected) + if want := string(b); got != want { + t.Errorf("%s test #%d %q, got vs want:\n----\n%s----\n%s----", filename, i, text, got, want) + continue + } + // Check that rendering and re-parsing results in an identical tree. + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(Render(pw, doc)) + }() + doc1, err := Parse(pr) + if err != nil { + t.Fatal(err) + } + got1, err := dump(doc1) + if err != nil { + t.Fatal(err) + } + if got != got1 { + t.Errorf("%s test #%d %q, got vs got1:\n----\n%s----\n%s----", filename, i, text, got, got1) + continue } } } diff --git a/libgo/go/html/render.go b/libgo/go/html/render.go new file mode 100644 index 0000000..e1ec66f --- /dev/null +++ b/libgo/go/html/render.go @@ -0,0 +1,169 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bufio" + "fmt" + "io" + "os" +) + +type writer interface { + io.Writer + WriteByte(byte) os.Error + WriteString(string) (int, os.Error) +} + +// Render renders the parse tree n to the given writer. +// +// For 'well-formed' parse trees, calling Parse on the output of Render will +// result in a clone of the original tree. +// +// 'Well-formed' is not formally specified, but calling Parse on arbitrary +// input results in a 'well-formed' parse tree if Parse does not return an +// error. Programmatically constructed trees are typically also 'well-formed', +// but it is possible to construct a tree that, when rendered and re-parsed, +// results in a different tree. A simple example is that a solitary text node +// would become a tree containing <html>, <head> and <body> elements. Another +// example is that the programmatic equivalent of "a<head>b</head>c" becomes +// "<html><head><head/><body>abc</body></html>". +// +// Comment nodes are elided from the output, analogous to Parse skipping over +// any <!--comment--> input. +func Render(w io.Writer, n *Node) os.Error { + if x, ok := w.(writer); ok { + return render(x, n) + } + buf := bufio.NewWriter(w) + if err := render(buf, n); err != nil { + return err + } + return buf.Flush() +} + +func render(w writer, n *Node) os.Error { + // Render non-element nodes; these are the easy cases. + switch n.Type { + case ErrorNode: + return os.NewError("html: cannot render an ErrorNode node") + case TextNode: + return escape(w, n.Data) + case DocumentNode: + for _, c := range n.Child { + if err := render(w, c); err != nil { + return err + } + } + return nil + case ElementNode: + // No-op. + case CommentNode: + return nil + case DoctypeNode: + if _, err := w.WriteString("<!DOCTYPE "); err != nil { + return err + } + if _, err := w.WriteString(n.Data); err != nil { + return err + } + return w.WriteByte('>') + default: + return os.NewError("html: unknown node type") + } + + // Render the <xxx> opening tag. + if err := w.WriteByte('<'); err != nil { + return err + } + if _, err := w.WriteString(n.Data); err != nil { + return err + } + for _, a := range n.Attr { + if err := w.WriteByte(' '); err != nil { + return err + } + if _, err := w.WriteString(a.Key); err != nil { + return err + } + if _, err := w.WriteString(`="`); err != nil { + return err + } + if err := escape(w, a.Val); err != nil { + return err + } + if err := w.WriteByte('"'); err != nil { + return err + } + } + if voidElements[n.Data] { + if len(n.Child) != 0 { + return fmt.Errorf("html: void element <%s> has child nodes", n.Data) + } + _, err := w.WriteString("/>") + return err + } + if err := w.WriteByte('>'); err != nil { + return err + } + + // Render any child nodes. + switch n.Data { + case "noembed", "noframes", "noscript", "script", "style": + for _, c := range n.Child { + if c.Type != TextNode { + return fmt.Errorf("html: raw text element <%s> has non-text child node", n.Data) + } + if _, err := w.WriteString(c.Data); err != nil { + return err + } + } + case "textarea", "title": + for _, c := range n.Child { + if c.Type != TextNode { + return fmt.Errorf("html: RCDATA element <%s> has non-text child node", n.Data) + } + if err := render(w, c); err != nil { + return err + } + } + default: + for _, c := range n.Child { + if err := render(w, c); err != nil { + return err + } + } + } + + // Render the </xxx> closing tag. + if _, err := w.WriteString("</"); err != nil { + return err + } + if _, err := w.WriteString(n.Data); err != nil { + return err + } + return w.WriteByte('>') +} + +// Section 13.1.2, "Elements", gives this list of void elements. Void elements +// are those that can't have any contents. +var voidElements = map[string]bool{ + "area": true, + "base": true, + "br": true, + "col": true, + "command": true, + "embed": true, + "hr": true, + "img": true, + "input": true, + "keygen": true, + "link": true, + "meta": true, + "param": true, + "source": true, + "track": true, + "wbr": true, +} diff --git a/libgo/go/html/render_test.go b/libgo/go/html/render_test.go new file mode 100644 index 0000000..d166a3b --- /dev/null +++ b/libgo/go/html/render_test.go @@ -0,0 +1,111 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "testing" +) + +func TestRenderer(t *testing.T) { + n := &Node{ + Type: ElementNode, + Data: "html", + Child: []*Node{ + &Node{ + Type: ElementNode, + Data: "head", + }, + &Node{ + Type: ElementNode, + Data: "body", + Child: []*Node{ + &Node{ + Type: TextNode, + Data: "0<1", + }, + &Node{ + Type: ElementNode, + Data: "p", + Attr: []Attribute{ + Attribute{ + Key: "id", + Val: "A", + }, + Attribute{ + Key: "foo", + Val: `abc"def`, + }, + }, + Child: []*Node{ + &Node{ + Type: TextNode, + Data: "2", + }, + &Node{ + Type: ElementNode, + Data: "b", + Attr: []Attribute{ + Attribute{ + Key: "empty", + Val: "", + }, + }, + Child: []*Node{ + &Node{ + Type: TextNode, + Data: "3", + }, + }, + }, + &Node{ + Type: ElementNode, + Data: "i", + Attr: []Attribute{ + Attribute{ + Key: "backslash", + Val: `\`, + }, + }, + Child: []*Node{ + &Node{ + Type: TextNode, + Data: "&4", + }, + }, + }, + }, + }, + &Node{ + Type: TextNode, + Data: "5", + }, + &Node{ + Type: ElementNode, + Data: "blockquote", + }, + &Node{ + Type: ElementNode, + Data: "br", + }, + &Node{ + Type: TextNode, + Data: "6", + }, + }, + }, + }, + } + want := `<html><head></head><body>0<1<p id="A" foo="abc"def">` + + `2<b empty="">3</b><i backslash="\">&4</i></p>` + + `5<blockquote></blockquote><br/>6</body></html>` + b := new(bytes.Buffer) + if err := Render(b, n); err != nil { + t.Fatal(err) + } + if got := b.String(); got != want { + t.Errorf("got vs want:\n%s\n%s\n", got, want) + } +} diff --git a/libgo/go/html/token.go b/libgo/go/html/token.go index d266b3a..2826f95 100644 --- a/libgo/go/html/token.go +++ b/libgo/go/html/token.go @@ -9,6 +9,7 @@ import ( "io" "os" "strconv" + "strings" ) // A TokenType is the type of a Token. @@ -100,13 +101,19 @@ func (t Token) String() string { case SelfClosingTagToken: return "<" + t.tagString() + "/>" case CommentToken: - return "<!--" + EscapeString(t.Data) + "-->" + return "<!--" + t.Data + "-->" case DoctypeToken: - return "<!DOCTYPE " + EscapeString(t.Data) + ">" + return "<!DOCTYPE " + t.Data + ">" } return "Invalid(" + strconv.Itoa(int(t.Type)) + ")" } +// span is a range of bytes in a Tokenizer's buffer. The start is inclusive, +// the end is exclusive. +type span struct { + start, end int +} + // A Tokenizer returns a stream of HTML Tokens. type Tokenizer struct { // If ReturnComments is set, Next returns comment tokens; @@ -115,7 +122,7 @@ type Tokenizer struct { // r is the source of the HTML text. r io.Reader - // tt is the TokenType of the most recently read token. + // tt is the TokenType of the current token. tt TokenType // err is the first error encountered during tokenization. It is possible // for tt != Error && err != nil to hold: this means that Next returned a @@ -125,10 +132,26 @@ type Tokenizer struct { // subsequent Next calls would return an ErrorToken. // err is never reset. Once it becomes non-nil, it stays non-nil. err os.Error - // buf[p0:p1] holds the raw data of the most recent token. - // buf[p1:] is buffered input that will yield future tokens. - p0, p1 int - buf []byte + // buf[raw.start:raw.end] holds the raw bytes of the current token. + // buf[raw.end:] is buffered input that will yield future tokens. + raw span + buf []byte + // buf[data.start:data.end] holds the raw bytes of the current token's data: + // a text token's text, a tag token's tag name, etc. + data span + // pendingAttr is the attribute key and value currently being tokenized. + // When complete, pendingAttr is pushed onto attr. nAttrReturned is + // incremented on each call to TagAttr. + pendingAttr [2]span + attr [][2]span + nAttrReturned int + // rawTag is the "script" in "</script>" that closes the next token. If + // non-empty, the subsequent call to Next will return a raw or RCDATA text + // token: one that treats "<p>" as text instead of an element. + // rawTag's contents are lower-cased. + rawTag string + // textIsRaw is whether the current text token's data is not escaped. + textIsRaw bool } // Error returns the error associated with the most recent ErrorToken token. @@ -140,33 +163,42 @@ func (z *Tokenizer) Error() os.Error { return z.err } -// Raw returns the unmodified text of the current token. Calling Next, Token, -// Text, TagName or TagAttr may change the contents of the returned slice. -func (z *Tokenizer) Raw() []byte { - return z.buf[z.p0:z.p1] -} - // readByte returns the next byte from the input stream, doing a buffered read -// from z.r into z.buf if necessary. z.buf[z.p0:z.p1] remains a contiguous byte +// from z.r into z.buf if necessary. z.buf[z.raw.start:z.raw.end] remains a contiguous byte // slice that holds all the bytes read so far for the current token. // It sets z.err if the underlying reader returns an error. // Pre-condition: z.err == nil. func (z *Tokenizer) readByte() byte { - if z.p1 >= len(z.buf) { + if z.raw.end >= len(z.buf) { // Our buffer is exhausted and we have to read from z.r. - // We copy z.buf[z.p0:z.p1] to the beginning of z.buf. If the length - // z.p1 - z.p0 is more than half the capacity of z.buf, then we + // We copy z.buf[z.raw.start:z.raw.end] to the beginning of z.buf. If the length + // z.raw.end - z.raw.start is more than half the capacity of z.buf, then we // allocate a new buffer before the copy. c := cap(z.buf) - d := z.p1 - z.p0 + d := z.raw.end - z.raw.start var buf1 []byte if 2*d > c { buf1 = make([]byte, d, 2*c) } else { buf1 = z.buf[:d] } - copy(buf1, z.buf[z.p0:z.p1]) - z.p0, z.p1, z.buf = 0, d, buf1[:d] + copy(buf1, z.buf[z.raw.start:z.raw.end]) + if x := z.raw.start; x != 0 { + // Adjust the data/attr spans to refer to the same contents after the copy. + z.data.start -= x + z.data.end -= x + z.pendingAttr[0].start -= x + z.pendingAttr[0].end -= x + z.pendingAttr[1].start -= x + z.pendingAttr[1].end -= x + for i := range z.attr { + z.attr[i][0].start -= x + z.attr[i][0].end -= x + z.attr[i][1].start -= x + z.attr[i][1].end -= x + } + } + z.raw.start, z.raw.end, z.buf = 0, d, buf1[:d] // Now that we have copied the live bytes to the start of the buffer, // we read from z.r into the remainder. n, err := z.r.Read(buf1[d:cap(buf1)]) @@ -176,297 +208,467 @@ func (z *Tokenizer) readByte() byte { } z.buf = buf1[:d+n] } - x := z.buf[z.p1] - z.p1++ + x := z.buf[z.raw.end] + z.raw.end++ return x } -// readTo keeps reading bytes until x is found or a read error occurs. If an -// error does occur, z.err is set to that error. -// Pre-condition: z.err == nil. -func (z *Tokenizer) readTo(x uint8) { +// skipWhiteSpace skips past any white space. +func (z *Tokenizer) skipWhiteSpace() { + if z.err != nil { + return + } for { c := z.readByte() if z.err != nil { return } switch c { - case x: + case ' ', '\n', '\r', '\t', '\f': + // No-op. + default: + z.raw.end-- return - case '\\': - z.readByte() + } + } +} + +// readRawOrRCDATA reads until the next "</foo>", where "foo" is z.rawTag and +// is typically something like "script" or "textarea". +func (z *Tokenizer) readRawOrRCDATA() { +loop: + for { + c := z.readByte() + if z.err != nil { + break loop + } + if c != '<' { + continue loop + } + c = z.readByte() + if z.err != nil { + break loop + } + if c != '/' { + continue loop + } + for i := 0; i < len(z.rawTag); i++ { + c = z.readByte() if z.err != nil { - return + break loop + } + if c != z.rawTag[i] && c != z.rawTag[i]-('a'-'A') { + continue loop } } + c = z.readByte() + if z.err != nil { + break loop + } + switch c { + case ' ', '\n', '\r', '\t', '\f', '/', '>': + // The 3 is 2 for the leading "</" plus 1 for the trailing character c. + z.raw.end -= 3 + len(z.rawTag) + break loop + case '<': + // Step back one, to catch "</foo</foo>". + z.raw.end-- + } } + z.data.end = z.raw.end + // A textarea's or title's RCDATA can contain escaped entities. + z.textIsRaw = z.rawTag != "textarea" && z.rawTag != "title" + z.rawTag = "" } -// nextComment reads the next token starting with "<!--". -// The opening "<!--" has already been consumed. -// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 4 <= z.p1. -func (z *Tokenizer) nextComment() { - // <!--> is a valid comment. +// readComment reads the next comment token starting with "<!--". The opening +// "<!--" has already been consumed. +func (z *Tokenizer) readComment() { + z.data.start = z.raw.end + defer func() { + if z.data.end < z.data.start { + // It's a comment with no data, like <!-->. + z.data.end = z.data.start + } + }() for dashCount := 2; ; { c := z.readByte() if z.err != nil { + z.data.end = z.raw.end return } switch c { case '-': dashCount++ + continue case '>': if dashCount >= 2 { - z.tt = CommentToken + z.data.end = z.raw.end - len("-->") return } - dashCount = 0 - default: - dashCount = 0 + case '!': + if dashCount >= 2 { + c = z.readByte() + if z.err != nil { + z.data.end = z.raw.end + return + } + if c == '>' { + z.data.end = z.raw.end - len("--!>") + return + } + } } + dashCount = 0 } } -// nextMarkupDeclaration reads the next token starting with "<!". -// It might be a "<!--comment-->", a "<!DOCTYPE foo>", or "<!malformed text". -// The opening "<!" has already been consumed. -// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 2 <= z.p1. -func (z *Tokenizer) nextMarkupDeclaration() { +// readUntilCloseAngle reads until the next ">". +func (z *Tokenizer) readUntilCloseAngle() { + z.data.start = z.raw.end + for { + c := z.readByte() + if z.err != nil { + z.data.end = z.raw.end + return + } + if c == '>' { + z.data.end = z.raw.end - len(">") + return + } + } +} + +// readMarkupDeclaration reads the next token starting with "<!". It might be +// a "<!--comment-->", a "<!DOCTYPE foo>", or "<!a bogus comment". The opening +// "<!" has already been consumed. +func (z *Tokenizer) readMarkupDeclaration() TokenType { + z.data.start = z.raw.end var c [2]byte for i := 0; i < 2; i++ { c[i] = z.readByte() if z.err != nil { - return + z.data.end = z.raw.end + return CommentToken } } if c[0] == '-' && c[1] == '-' { - z.nextComment() - return + z.readComment() + return CommentToken } - z.p1 -= 2 - const s = "DOCTYPE " - for i := 0; ; i++ { + z.raw.end -= 2 + const s = "DOCTYPE" + for i := 0; i < len(s); i++ { c := z.readByte() if z.err != nil { - return + z.data.end = z.raw.end + return CommentToken } - // Capitalize c. - if 'a' <= c && c <= 'z' { - c = 'A' + (c - 'a') - } - if i < len(s) && c != s[i] { - z.nextText() - return - } - if c == '>' { - if i >= len(s) { - z.tt = DoctypeToken - } - return + if c != s[i] && c != s[i]+('a'-'A') { + // Back up to read the fragment of "DOCTYPE" again. + z.raw.end = z.data.start + z.readUntilCloseAngle() + return CommentToken } } + if z.skipWhiteSpace(); z.err != nil { + z.data.start = z.raw.end + z.data.end = z.raw.end + return DoctypeToken + } + z.readUntilCloseAngle() + return DoctypeToken } -// nextTag reads the next token starting with "<". It might be a "<startTag>", -// an "</endTag>", a "<!markup declaration>", or "<malformed text". -// The opening "<" has already been consumed. -// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 1 <= z.p1. -func (z *Tokenizer) nextTag() { - c := z.readByte() - if z.err != nil { - return - } - switch { - case c == '/': - z.tt = EndTagToken - // Lower-cased characters are more common in tag names, so we check for them first. - case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': - z.tt = StartTagToken - case c == '!': - z.nextMarkupDeclaration() - return - case c == '?': - z.tt, z.err = ErrorToken, os.NewError("html: TODO: implement XML processing instructions") - return - default: - z.tt, z.err = ErrorToken, os.NewError("html: TODO: handle malformed tags") - return +// readStartTag reads the next start tag token. The opening "<a" has already +// been consumed, where 'a' means anything in [A-Za-z]. +func (z *Tokenizer) readStartTag() TokenType { + z.attr = z.attr[:0] + z.nAttrReturned = 0 + // Read the tag name and attribute key/value pairs. + z.readTagName() + if z.skipWhiteSpace(); z.err != nil { + return ErrorToken } for { c := z.readByte() - if z.err != nil { - return + if z.err != nil || c == '>' { + break } - switch c { - case '"', '\'': - z.readTo(c) - if z.err != nil { - return - } - case '>': - if z.buf[z.p1-2] == '/' && z.tt == StartTagToken { - z.tt = SelfClosingTagToken + z.raw.end-- + z.readTagAttrKey() + z.readTagAttrVal() + // Save pendingAttr if it has a non-empty key. + if z.pendingAttr[0].start != z.pendingAttr[0].end { + z.attr = append(z.attr, z.pendingAttr) + } + if z.skipWhiteSpace(); z.err != nil { + break + } + } + // Any "<noembed>", "<noframes>", "<noscript>", "<script>", "<style>", + // "<textarea>" or "<title>" tag flags the tokenizer's next token as raw. + // The tag name lengths of these special cases ranges in [5, 8]. + if x := z.data.end - z.data.start; 5 <= x && x <= 8 { + switch z.buf[z.data.start] { + case 'n', 's', 't', 'N', 'S', 'T': + switch s := strings.ToLower(string(z.buf[z.data.start:z.data.end])); s { + case "noembed", "noframes", "noscript", "script", "style", "textarea", "title": + z.rawTag = s } + } + } + // Look for a self-closing token like "<br/>". + if z.err == nil && z.buf[z.raw.end-2] == '/' { + return SelfClosingTagToken + } + return StartTagToken +} + +// readEndTag reads the next end tag token. The opening "</a" has already +// been consumed, where 'a' means anything in [A-Za-z]. +func (z *Tokenizer) readEndTag() { + z.attr = z.attr[:0] + z.nAttrReturned = 0 + z.readTagName() + for { + c := z.readByte() + if z.err != nil || c == '>' { return } } } -// nextText reads all text up until an '<'. -// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 1 <= z.p1. -func (z *Tokenizer) nextText() { +// readTagName sets z.data to the "div" in "<div k=v>". The reader (z.raw.end) +// is positioned such that the first byte of the tag name (the "d" in "<div") +// has already been consumed. +func (z *Tokenizer) readTagName() { + z.data.start = z.raw.end - 1 for { c := z.readByte() if z.err != nil { + z.data.end = z.raw.end return } - if c == '<' { - z.p1-- + switch c { + case ' ', '\n', '\r', '\t', '\f': + z.data.end = z.raw.end - 1 + return + case '/', '>': + z.raw.end-- + z.data.end = z.raw.end return } } } -// Next scans the next token and returns its type. -func (z *Tokenizer) Next() TokenType { +// readTagAttrKey sets z.pendingAttr[0] to the "k" in "<div k=v>". +// Precondition: z.err == nil. +func (z *Tokenizer) readTagAttrKey() { + z.pendingAttr[0].start = z.raw.end for { - if z.err != nil { - z.tt = ErrorToken - return z.tt - } - z.p0 = z.p1 c := z.readByte() if z.err != nil { - z.tt = ErrorToken - return z.tt + z.pendingAttr[0].end = z.raw.end + return } - // We assume that the next token is text unless proven otherwise. - z.tt = TextToken - if c != '<' { - z.nextText() - } else { - z.nextTag() - if z.tt == CommentToken && !z.ReturnComments { - continue - } + switch c { + case ' ', '\n', '\r', '\t', '\f', '/': + z.pendingAttr[0].end = z.raw.end - 1 + return + case '=', '>': + z.raw.end-- + z.pendingAttr[0].end = z.raw.end + return } - return z.tt } - panic("unreachable") } -// trim returns the largest j such that z.buf[i:j] contains only white space, -// or only white space plus the final ">" or "/>" of the raw data. -func (z *Tokenizer) trim(i int) int { - k := z.p1 - for ; i < k; i++ { - switch z.buf[i] { - case ' ', '\n', '\t', '\f': - continue - case '>': - if i == k-1 { - return k +// readTagAttrVal sets z.pendingAttr[1] to the "v" in "<div k=v>". +func (z *Tokenizer) readTagAttrVal() { + z.pendingAttr[1].start = z.raw.end + z.pendingAttr[1].end = z.raw.end + if z.skipWhiteSpace(); z.err != nil { + return + } + c := z.readByte() + if z.err != nil { + return + } + if c != '=' { + z.raw.end-- + return + } + if z.skipWhiteSpace(); z.err != nil { + return + } + quote := z.readByte() + if z.err != nil { + return + } + switch quote { + case '>': + z.raw.end-- + return + + case '\'', '"': + z.pendingAttr[1].start = z.raw.end + for { + c := z.readByte() + if z.err != nil { + z.pendingAttr[1].end = z.raw.end + return } - case '/': - if i == k-2 { - return k + if c == quote { + z.pendingAttr[1].end = z.raw.end - 1 + return } } - return i - } - return k -} -// tagName finds the tag name at the start of z.buf[i:] and returns that name -// lower-cased, as well as the trimmed cursor location afterwards. -func (z *Tokenizer) tagName(i int) ([]byte, int) { - i0 := i -loop: - for ; i < z.p1; i++ { - c := z.buf[i] - switch c { - case ' ', '\n', '\t', '\f', '/', '>': - break loop - } - if 'A' <= c && c <= 'Z' { - z.buf[i] = c + 'a' - 'A' + default: + z.pendingAttr[1].start = z.raw.end - 1 + for { + c := z.readByte() + if z.err != nil { + z.pendingAttr[1].end = z.raw.end + return + } + switch c { + case ' ', '\n', '\r', '\t', '\f': + z.pendingAttr[1].end = z.raw.end - 1 + return + case '>': + z.raw.end-- + z.pendingAttr[1].end = z.raw.end + return + } } } - return z.buf[i0:i], z.trim(i) } -// unquotedAttrVal finds the unquoted attribute value at the start of z.buf[i:] -// and returns that value, as well as the trimmed cursor location afterwards. -func (z *Tokenizer) unquotedAttrVal(i int) ([]byte, int) { - i0 := i +// next scans the next token and returns its type. +func (z *Tokenizer) next() TokenType { + if z.err != nil { + return ErrorToken + } + z.raw.start = z.raw.end + z.data.start = z.raw.end + z.data.end = z.raw.end + if z.rawTag != "" { + z.readRawOrRCDATA() + return TextToken + } + z.textIsRaw = false + loop: - for ; i < z.p1; i++ { - switch z.buf[i] { - case ' ', '\n', '\t', '\f', '>': + for { + c := z.readByte() + if z.err != nil { break loop - case '&': - // TODO: unescape the entity. } - } - return z.buf[i0:i], z.trim(i) -} - -// attrName finds the largest attribute name at the start -// of z.buf[i:] and returns it lower-cased, as well -// as the trimmed cursor location after that name. -// -// http://dev.w3.org/html5/spec/Overview.html#syntax-attribute-name -// TODO: unicode characters -func (z *Tokenizer) attrName(i int) ([]byte, int) { - for z.buf[i] == '/' { - i++ - if z.buf[i] == '>' { - return nil, z.trim(i) + if c != '<' { + continue loop } - } - i0 := i -loop: - for ; i < z.p1; i++ { - c := z.buf[i] - switch c { - case '>', '/', '=': + + // Check if the '<' we have just read is part of a tag, comment + // or doctype. If not, it's part of the accumulated text token. + c = z.readByte() + if z.err != nil { break loop } + var tokenType TokenType switch { - case 'A' <= c && c <= 'Z': - z.buf[i] = c + 'a' - 'A' - case c > ' ' && c < 0x7f: - // No-op. + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': + tokenType = StartTagToken + case c == '/': + tokenType = EndTagToken + case c == '!' || c == '?': + // We use CommentToken to mean any of "<!--actual comments-->", + // "<!DOCTYPE declarations>" and "<?xml processing instructions?>". + tokenType = CommentToken default: - break loop + continue + } + + // We have a non-text token, but we might have accumulated some text + // before that. If so, we return the text first, and return the non- + // text token on the subsequent call to Next. + if x := z.raw.end - len("<a"); z.raw.start < x { + z.raw.end = x + z.data.end = x + return TextToken + } + switch tokenType { + case StartTagToken: + return z.readStartTag() + case EndTagToken: + c = z.readByte() + if z.err != nil { + break loop + } + if c == '>' { + // "</>" does not generate a token at all. + // Reset the tokenizer state and start again. + z.raw.start = z.raw.end + z.data.start = z.raw.end + z.data.end = z.raw.end + continue loop + } + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { + z.readEndTag() + return EndTagToken + } + z.raw.end-- + z.readUntilCloseAngle() + return CommentToken + case CommentToken: + if c == '!' { + return z.readMarkupDeclaration() + } + z.raw.end-- + z.readUntilCloseAngle() + return CommentToken + } + } + if z.raw.start < z.raw.end { + z.data.end = z.raw.end + return TextToken + } + return ErrorToken +} + +// Next scans the next token and returns its type. +func (z *Tokenizer) Next() TokenType { + for { + z.tt = z.next() + // TODO: remove the ReturnComments option. A tokenizer should + // always return comment tags. + if z.tt == CommentToken && !z.ReturnComments { + continue } + return z.tt } - return z.buf[i0:i], z.trim(i) + panic("unreachable") +} + +// Raw returns the unmodified text of the current token. Calling Next, Token, +// Text, TagName or TagAttr may change the contents of the returned slice. +func (z *Tokenizer) Raw() []byte { + return z.buf[z.raw.start:z.raw.end] } // Text returns the unescaped text of a text, comment or doctype token. The // contents of the returned slice may change on the next call to Next. func (z *Tokenizer) Text() []byte { - var i0, i1 int switch z.tt { - case TextToken: - i0 = z.p0 - i1 = z.p1 - case CommentToken: - // Trim the "<!--" from the left and the "-->" from the right. - // "<!-->" is a valid comment, so the adjusted endpoints might overlap. - i0 = z.p0 + 4 - i1 = z.p1 - 3 - case DoctypeToken: - // Trim the "<!DOCTYPE " from the left and the ">" from the right. - i0 = z.p0 + 10 - i1 = z.p1 - 1 - default: - return nil - } - z.p0 = z.p1 - if i0 < i1 { - return unescape(z.buf[i0:i1]) + case TextToken, CommentToken, DoctypeToken: + s := z.buf[z.data.start:z.data.end] + z.data.start = z.raw.end + z.data.end = z.raw.end + if !z.textIsRaw { + s = unescape(s) + } + return s } return nil } @@ -475,73 +677,33 @@ func (z *Tokenizer) Text() []byte { // `<IMG SRC="foo">`) and whether the tag has attributes. // The contents of the returned slice may change on the next call to Next. func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { - i := z.p0 + 1 - if i >= z.p1 { - z.p0 = z.p1 - return nil, false - } - if z.buf[i] == '/' { - i++ + if z.data.start < z.data.end { + switch z.tt { + case StartTagToken, EndTagToken, SelfClosingTagToken: + s := z.buf[z.data.start:z.data.end] + z.data.start = z.raw.end + z.data.end = z.raw.end + return lower(s), z.nAttrReturned < len(z.attr) + } } - name, z.p0 = z.tagName(i) - hasAttr = z.p0 != z.p1 - return + return nil, false } // TagAttr returns the lower-cased key and unescaped value of the next unparsed // attribute for the current tag token and whether there are more attributes. // The contents of the returned slices may change on the next call to Next. func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { - key, i := z.attrName(z.p0) - // Check for an empty attribute value. - if i == z.p1 { - z.p0 = i - return - } - // Get past the equals and quote characters. - if z.buf[i] != '=' { - z.p0, moreAttr = i, true - return - } - i = z.trim(i + 1) - if i == z.p1 { - z.p0 = i - return - } - closeQuote := z.buf[i] - if closeQuote != '\'' && closeQuote != '"' { - val, z.p0 = z.unquotedAttrVal(i) - moreAttr = z.p0 != z.p1 - return - } - i = z.trim(i + 1) - // Copy and unescape everything up to the closing quote. - dst, src := i, i -loop: - for src < z.p1 { - c := z.buf[src] - switch c { - case closeQuote: - src++ - break loop - case '&': - dst, src = unescapeEntity(z.buf, dst, src, true) - case '\\': - if src == z.p1 { - z.buf[dst] = '\\' - dst++ - } else { - z.buf[dst] = z.buf[src+1] - dst, src = dst+1, src+2 - } - default: - z.buf[dst] = c - dst, src = dst+1, src+1 + if z.nAttrReturned < len(z.attr) { + switch z.tt { + case StartTagToken, SelfClosingTagToken: + x := z.attr[z.nAttrReturned] + z.nAttrReturned++ + key = z.buf[x[0].start:x[0].end] + val = z.buf[x[1].start:x[1].end] + return lower(key), unescape(val), z.nAttrReturned < len(z.attr) } } - val, z.p0 = z.buf[i:dst], z.trim(src) - moreAttr = z.p0 != z.p1 - return + return nil, nil, false } // Token returns the next Token. The result's Data and Attr values remain valid @@ -551,7 +713,7 @@ func (z *Tokenizer) Token() Token { switch z.tt { case TextToken, CommentToken, DoctypeToken: t.Data = string(z.Text()) - case StartTagToken, EndTagToken, SelfClosingTagToken: + case StartTagToken, SelfClosingTagToken: var attr []Attribute name, moreAttr := z.TagName() for moreAttr { @@ -561,6 +723,9 @@ func (z *Tokenizer) Token() Token { } t.Data = string(name) t.Attr = attr + case EndTagToken: + name, _ := z.TagName() + t.Data = string(name) } return t } diff --git a/libgo/go/html/token_test.go b/libgo/go/html/token_test.go index 0a0beb2..310cd97 100644 --- a/libgo/go/html/token_test.go +++ b/libgo/go/html/token_test.go @@ -21,6 +21,11 @@ type tokenTest struct { } var tokenTests = []tokenTest{ + { + "empty", + "", + "", + }, // A single text node. The tokenizer should not break text nodes on whitespace, // nor should it normalize whitespace within a text node. { @@ -41,6 +46,88 @@ var tokenTests = []tokenTest{ "<a>b<c/>d</e>", "<a>$b$<c/>$d$</e>", }, + // Angle brackets that aren't a tag. + { + "not a tag #0", + "<", + "<", + }, + { + "not a tag #1", + "</", + "</", + }, + { + "not a tag #2", + "</>", + "", + }, + { + "not a tag #3", + "a</>b", + "a$b", + }, + { + "not a tag #4", + "</ >", + "<!-- -->", + }, + { + "not a tag #5", + "</.", + "<!--.-->", + }, + { + "not a tag #6", + "</.>", + "<!--.-->", + }, + { + "not a tag #7", + "a < b", + "a < b", + }, + { + "not a tag #8", + "<.>", + "<.>", + }, + { + "not a tag #9", + "a<<<b>>>c", + "a<<$<b>$>>c", + }, + { + "not a tag #10", + "if x<0 and y < 0 then x*y>0", + "if x<0 and y < 0 then x*y>0", + }, + // EOF in a tag name. + { + "tag name eof #0", + "<a", + "", + }, + { + "tag name eof #1", + "<a ", + "", + }, + { + "tag name eof #2", + "a<b", + "a", + }, + { + "tag name eof #3", + "<a><b", + "<a>", + }, + { + "tag name eof #4", + `<a x`, + `<a x="">`, + }, // Some malformed tags that are missing a '>'. { "malformed tag #0", @@ -54,70 +141,198 @@ var tokenTests = []tokenTest{ }, { "malformed tag #2", + `<p id`, + `<p id="">`, + }, + { + "malformed tag #3", + `<p id=`, + `<p id="">`, + }, + { + "malformed tag #4", + `<p id=>`, + `<p id="">`, + }, + { + "malformed tag #5", + `<p id=0`, + `<p id="0">`, + }, + { + "malformed tag #6", `<p id=0</p>`, `<p id="0</p">`, }, { - "malformed tag #3", + "malformed tag #7", `<p id="0</p>`, `<p id="0</p>">`, }, { - "malformed tag #4", + "malformed tag #8", `<p id="0"</p>`, `<p id="0" <="" p="">`, }, + // Raw text and RCDATA. + { + "basic raw text", + "<script><a></b></script>", + "<script>$<a></b>$</script>", + }, + { + "unfinished script end tag", + "<SCRIPT>a</SCR", + "<script>$a</SCR", + }, + { + "broken script end tag", + "<SCRIPT>a</SCR ipt>", + "<script>$a</SCR ipt>", + }, + { + "EOF in script end tag", + "<SCRIPT>a</SCRipt", + "<script>$a</SCRipt", + }, + { + "scriptx end tag", + "<SCRIPT>a</SCRiptx", + "<script>$a</SCRiptx", + }, + { + "' ' completes script end tag", + "<SCRIPT>a</SCRipt ", + "<script>$a$</script>", + }, + { + "'>' completes script end tag", + "<SCRIPT>a</SCRipt>", + "<script>$a$</script>", + }, + { + "self-closing script end tag", + "<SCRIPT>a</SCRipt/>", + "<script>$a$</script>", + }, + { + "nested script tag", + "<SCRIPT>a</SCRipt<script>", + "<script>$a</SCRipt<script>", + }, + { + "script end tag after unfinished", + "<SCRIPT>a</SCRipt</script>", + "<script>$a</SCRipt$</script>", + }, + { + "script/style mismatched tags", + "<script>a</style>", + "<script>$a</style>", + }, + { + "style element with entity", + "<style>'", + "<style>$&apos;", + }, + { + "textarea with tag", + "<textarea><div></textarea>", + "<textarea>$<div>$</textarea>", + }, + { + "title with tag and entity", + "<title><b>K&R C</b></title>", + "<title>$<b>K&R C</b>$</title>", + }, + // DOCTYPE tests. + { + "Proper DOCTYPE", + "<!DOCTYPE html>", + "<!DOCTYPE html>", + }, + { + "DOCTYPE with no space", + "<!doctypehtml>", + "<!DOCTYPE html>", + }, + { + "DOCTYPE with two spaces", + "<!doctype html>", + "<!DOCTYPE html>", + }, + { + "looks like DOCTYPE but isn't", + "<!DOCUMENT html>", + "<!--DOCUMENT html-->", + }, + { + "DOCTYPE at EOF", + "<!DOCtype", + "<!DOCTYPE >", + }, + // XML processing instructions. + { + "XML processing instruction", + "<?xml?>", + "<!--?xml?-->", + }, // Comments. { "comment0", "abc<b><!-- skipme --></b>def", - "abc$<b>$</b>$def", + "abc$<b>$<!-- skipme -->$</b>$def", }, { "comment1", "a<!-->z", - "a$z", + "a$<!---->$z", }, { "comment2", "a<!--->z", - "a$z", + "a$<!---->$z", }, { "comment3", "a<!--x>-->z", - "a$z", + "a$<!--x>-->$z", }, { "comment4", "a<!--x->-->z", - "a$z", + "a$<!--x->-->$z", }, { "comment5", "a<!>z", - "a$<!>z", + "a$<!---->$z", }, { "comment6", "a<!->z", - "a$<!->z", + "a$<!----->$z", }, { "comment7", "a<!---<>z", - "a$<!---<>z", + "a$<!---<>z-->", }, { "comment8", "a<!--z", - "a$<!--z", + "a$<!--z-->", + }, + { + "comment9", + "a<!--x--!>z", + "a$<!--x-->$z", }, // An attribute with a backslash. { "backslash", `<p id="a\"b">`, - `<p id="a"b">`, + `<p id="a\" b"="">`, }, // Entities, tag name and attribute key lower-casing, and whitespace // normalization within a tag. @@ -133,11 +348,14 @@ var tokenTests = []tokenTest{ `<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`, `<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`, }, - { - "entity without semicolon", - `¬it;∉<a b="q=z&=5¬ice=hello¬=world">`, - `¬it;∉$<a b="q=z&amp=5&notice=hello¬=world">`, - }, + /* + // TODO: re-enable this test when it works. This input/output matches html5lib's behavior. + { + "entity without semicolon", + `¬it;∉<a b="q=z&=5¬ice=hello¬=world">`, + `¬it;∉$<a b="q=z&amp=5&notice=hello¬=world">`, + }, + */ { "entity with digits", "½", @@ -190,21 +408,34 @@ var tokenTests = []tokenTest{ `<meta http-equiv="content-type">`, `<meta http-equiv="content-type">`, }, + { + "Mixed attributes", + `a<P V="0 1" w='2' X=3 y>z`, + `a$<p v="0 1" w="2" x="3" y="">$z`, + }, + { + "Attributes with a solitary single quote", + `<p id=can't><p id=won't>`, + `<p id="can't">$<p id="won't">`, + }, } func TestTokenizer(t *testing.T) { loop: for _, tt := range tokenTests { - z := NewTokenizer(bytes.NewBuffer([]byte(tt.html))) - for i, s := range strings.Split(tt.golden, "$") { - if z.Next() == ErrorToken { - t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error()) - continue loop - } - actual := z.Token().String() - if s != actual { - t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual) - continue loop + z := NewTokenizer(strings.NewReader(tt.html)) + z.ReturnComments = true + if tt.golden != "" { + for i, s := range strings.Split(tt.golden, "$") { + if z.Next() == ErrorToken { + t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error()) + continue loop + } + actual := z.Token().String() + if s != actual { + t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual) + continue loop + } } } z.Next() diff --git a/libgo/go/http/cgi/child.go b/libgo/go/http/cgi/child.go index 8d0eca8..bf14c04 100644 --- a/libgo/go/http/cgi/child.go +++ b/libgo/go/http/cgi/child.go @@ -93,20 +93,20 @@ func RequestFromMap(params map[string]string) (*http.Request, os.Error) { if r.Host != "" { // Hostname is provided, so we can reasonably construct a URL, // even if we have to assume 'http' for the scheme. - r.RawURL = "http://" + r.Host + params["REQUEST_URI"] - url, err := url.Parse(r.RawURL) + rawurl := "http://" + r.Host + params["REQUEST_URI"] + url, err := url.Parse(rawurl) if err != nil { - return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL) + return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + rawurl) } r.URL = url } // Fallback logic if we don't have a Host header or the URL // failed to parse if r.URL == nil { - r.RawURL = params["REQUEST_URI"] - url, err := url.Parse(r.RawURL) + uriStr := params["REQUEST_URI"] + url, err := url.Parse(uriStr) if err != nil { - return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL) + return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + uriStr) } r.URL = url } diff --git a/libgo/go/http/cgi/child_test.go b/libgo/go/http/cgi/child_test.go index eee043b..ec53ab8 100644 --- a/libgo/go/http/cgi/child_test.go +++ b/libgo/go/http/cgi/child_test.go @@ -49,9 +49,6 @@ func TestRequest(t *testing.T) { if g, e := req.Header.Get("Foo-Bar"), "baz"; e != g { t.Errorf("expected Foo-Bar %q; got %q", e, g) } - if g, e := req.RawURL, "http://example.com/path?a=b"; e != g { - t.Errorf("expected RawURL %q; got %q", e, g) - } if g, e := req.URL.String(), "http://example.com/path?a=b"; e != g { t.Errorf("expected URL %q; got %q", e, g) } @@ -81,9 +78,6 @@ func TestRequestWithoutHost(t *testing.T) { if err != nil { t.Fatalf("RequestFromMap: %v", err) } - if g, e := req.RawURL, "/path?a=b"; e != g { - t.Errorf("expected RawURL %q; got %q", e, g) - } if req.URL == nil { t.Fatalf("unexpected nil URL") } diff --git a/libgo/go/http/cgi/host.go b/libgo/go/http/cgi/host.go index f7de89f..9ea4c9d 100644 --- a/libgo/go/http/cgi/host.go +++ b/libgo/go/http/cgi/host.go @@ -32,13 +32,14 @@ import ( var trailingPort = regexp.MustCompile(`:([0-9]+)$`) var osDefaultInheritEnv = map[string][]string{ - "darwin": []string{"DYLD_LIBRARY_PATH"}, - "freebsd": []string{"LD_LIBRARY_PATH"}, - "hpux": []string{"LD_LIBRARY_PATH", "SHLIB_PATH"}, - "irix": []string{"LD_LIBRARY_PATH", "LD_LIBRARYN32_PATH", "LD_LIBRARY64_PATH"}, - "linux": []string{"LD_LIBRARY_PATH"}, - "solaris": []string{"LD_LIBRARY_PATH", "LD_LIBRARY_PATH_32", "LD_LIBRARY_PATH_64"}, - "windows": []string{"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"}, + "darwin": {"DYLD_LIBRARY_PATH"}, + "freebsd": {"LD_LIBRARY_PATH"}, + "hpux": {"LD_LIBRARY_PATH", "SHLIB_PATH"}, + "irix": {"LD_LIBRARY_PATH", "LD_LIBRARYN32_PATH", "LD_LIBRARY64_PATH"}, + "linux": {"LD_LIBRARY_PATH"}, + "openbsd": {"LD_LIBRARY_PATH"}, + "solaris": {"LD_LIBRARY_PATH", "LD_LIBRARY_PATH_32", "LD_LIBRARY_PATH_64"}, + "windows": {"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"}, } // Handler runs an executable in a subprocess with a CGI environment. @@ -68,6 +69,31 @@ type Handler struct { PathLocationHandler http.Handler } +// removeLeadingDuplicates remove leading duplicate in environments. +// It's possible to override environment like following. +// cgi.Handler{ +// ... +// Env: []string{"SCRIPT_FILENAME=foo.php"}, +// } +func removeLeadingDuplicates(env []string) (ret []string) { + n := len(env) + for i := 0; i < n; i++ { + e := env[i] + s := strings.SplitN(e, "=", 2)[0] + found := false + for j := i + 1; j < n; j++ { + if s == strings.SplitN(env[j], "=", 2)[0] { + found = true + break + } + } + if !found { + ret = append(ret, e) + } + } + return +} + func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { root := h.Root if root == "" { @@ -149,6 +175,8 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } } + env = removeLeadingDuplicates(env) + var cwd, path string if h.Dir != "" { path = h.Path @@ -294,7 +322,6 @@ func (h *Handler) handleInternalRedirect(rw http.ResponseWriter, req *http.Reque newReq := &http.Request{ Method: "GET", URL: url, - RawURL: path, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, diff --git a/libgo/go/http/cgi/host_test.go b/libgo/go/http/cgi/host_test.go index ff46631..6c0f1a0 100644 --- a/libgo/go/http/cgi/host_test.go +++ b/libgo/go/http/cgi/host_test.go @@ -451,3 +451,32 @@ func TestDirWindows(t *testing.T) { } runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) } + +func TestEnvOverride(t *testing.T) { + cgifile, _ := filepath.Abs("testdata/test.cgi") + + var perl string + var err os.Error + perl, err = exec.LookPath("perl") + if err != nil { + return + } + perl, _ = filepath.Abs(perl) + + cwd, _ := os.Getwd() + h := &Handler{ + Path: perl, + Root: "/test.cgi", + Dir: cwd, + Args: []string{cgifile}, + Env: []string{ + "SCRIPT_FILENAME=" + cgifile, + "REQUEST_URI=/foo/bar"}, + } + expectedMap := map[string]string{ + "cwd": cwd, + "env-SCRIPT_FILENAME": cgifile, + "env-REQUEST_URI": "/foo/bar", + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) +} diff --git a/libgo/go/http/chunked.go b/libgo/go/http/chunked.go index 6c23e69..eff9ae2 100644 --- a/libgo/go/http/chunked.go +++ b/libgo/go/http/chunked.go @@ -5,11 +5,11 @@ package http import ( + "bufio" "io" "log" "os" "strconv" - "bufio" ) // NewChunkedWriter returns a new writer that translates writes into HTTP diff --git a/libgo/go/http/client.go b/libgo/go/http/client.go index 44b3443..3fa4a05 100644 --- a/libgo/go/http/client.go +++ b/libgo/go/http/client.go @@ -56,9 +56,10 @@ type RoundTripper interface { // higher-level protocol details such as redirects, // authentication, or cookies. // - // RoundTrip may modify the request. The request Headers field is - // guaranteed to be initialized. - RoundTrip(req *Request) (resp *Response, err os.Error) + // RoundTrip should not modify the request, except for + // consuming the Body. The request's URL and Header fields + // are guaranteed to be initialized. + RoundTrip(*Request) (*Response, os.Error) } // Given a string of the form "host", "host:port", or "[ipv6::address]:port", @@ -76,7 +77,12 @@ type readClose struct { // Do sends an HTTP request and returns an HTTP response, following // policy (e.g. redirects, cookies, auth) as configured on the client. // -// Callers should close resp.Body when done reading from it. +// A non-nil response always contains a non-nil resp.Body. +// +// Callers should close resp.Body when done reading from it. If +// resp.Body is not closed, the Client's underlying RoundTripper +// (typically Transport) may not be able to re-use a persistent TCP +// connection to the server for a subsequent "keep-alive" request. // // Generally Get, Post, or PostForm will be used instead of Do. func (c *Client) Do(req *Request) (resp *Response, err os.Error) { @@ -91,11 +97,15 @@ func send(req *Request, t RoundTripper) (resp *Response, err os.Error) { if t == nil { t = DefaultTransport if t == nil { - err = os.NewError("no http.Client.Transport or http.DefaultTransport") + err = os.NewError("http: no Client.Transport or DefaultTransport") return } } + if req.URL == nil { + return nil, os.NewError("http: nil Request.URL") + } + // Most the callers of send (Get, Post, et al) don't need // Headers, leaving it uninitialized. We guarantee to the // Transport that this has been initialized, though. @@ -105,9 +115,6 @@ func send(req *Request, t RoundTripper) (resp *Response, err os.Error) { info := req.URL.RawUserinfo if len(info) > 0 { - if req.Header == nil { - req.Header = make(Header) - } req.Header.Set("Authorization", "Basic "+base64.URLEncoding.EncodeToString([]byte(info))) } return t.RoundTrip(req) @@ -166,6 +173,10 @@ func (c *Client) doFollowingRedirects(ireq *Request) (r *Response, err os.Error) } var via []*Request + if ireq.URL == nil { + return nil, os.NewError("http: nil Request.URL") + } + req := ireq urlStr := "" // next relative or absolute URL to fetch (after first request) for redirect := 0; ; redirect++ { diff --git a/libgo/go/http/client_test.go b/libgo/go/http/client_test.go index 8efb1d9..0ad6cd7 100644 --- a/libgo/go/http/client_test.go +++ b/libgo/go/http/client_test.go @@ -132,7 +132,9 @@ func TestPostFormRequestFormat(t *testing.T) { if tr.req.Close { t.Error("got Close true, want false") } - expectedBody := "bar=baz&foo=bar&foo=bar2" + // Depending on map iteration, body can be either of these. + expectedBody := "foo=bar&foo=bar2&bar=baz" + expectedBody1 := "bar=baz&foo=bar&foo=bar2" if g, e := tr.req.ContentLength, int64(len(expectedBody)); g != e { t.Errorf("got ContentLength %d, want %d", g, e) } @@ -140,8 +142,8 @@ func TestPostFormRequestFormat(t *testing.T) { if err != nil { t.Fatalf("ReadAll on req.Body: %v", err) } - if g := string(bodyb); g != expectedBody { - t.Errorf("got body %q, want %q", g, expectedBody) + if g := string(bodyb); g != expectedBody && g != expectedBody1 { + t.Errorf("got body %q, want %q or %q", g, expectedBody, expectedBody1) } } diff --git a/libgo/go/http/cookie.go b/libgo/go/http/cookie.go index fe70431..6935014 100644 --- a/libgo/go/http/cookie.go +++ b/libgo/go/http/cookie.go @@ -207,17 +207,16 @@ func readCookies(h Header, filter string) []*Cookie { return cookies } +var cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-") + func sanitizeName(n string) string { - n = strings.Replace(n, "\n", "-", -1) - n = strings.Replace(n, "\r", "-", -1) - return n + return cookieNameSanitizer.Replace(n) } +var cookieValueSanitizer = strings.NewReplacer("\n", " ", "\r", " ", ";", " ") + func sanitizeValue(v string) string { - v = strings.Replace(v, "\n", " ", -1) - v = strings.Replace(v, "\r", " ", -1) - v = strings.Replace(v, ";", " ", -1) - return v + return cookieValueSanitizer.Replace(v) } func unquoteCookieValue(v string) string { diff --git a/libgo/go/http/cookie_test.go b/libgo/go/http/cookie_test.go index d7aeda0..5de6aab 100644 --- a/libgo/go/http/cookie_test.go +++ b/libgo/go/http/cookie_test.go @@ -124,7 +124,7 @@ var readSetCookiesTests = []struct { Path: "/", Domain: ".google.ch", HttpOnly: true, - Expires: time.Time{Year: 2011, Month: 11, Day: 23, Hour: 1, Minute: 5, Second: 3, Weekday: 3, ZoneOffset: 0, Zone: "GMT"}, + Expires: time.Time{Year: 2011, Month: 11, Day: 23, Hour: 1, Minute: 5, Second: 3, ZoneOffset: 0, Zone: "GMT"}, RawExpires: "Wed, 23-Nov-2011 01:05:03 GMT", Raw: "NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly", }}, diff --git a/libgo/go/http/dump.go b/libgo/go/http/dump.go index 358980f..f78df57 100644 --- a/libgo/go/http/dump.go +++ b/libgo/go/http/dump.go @@ -44,7 +44,7 @@ func DumpRequest(req *Request, body bool) (dump []byte, err os.Error) { return } } - err = req.Write(&b) + err = req.dumpWrite(&b) req.Body = save if err != nil { return diff --git a/libgo/go/http/fcgi/child.go b/libgo/go/http/fcgi/child.go index 1971882..61dd3fb 100644 --- a/libgo/go/http/fcgi/child.go +++ b/libgo/go/http/fcgi/child.go @@ -194,7 +194,7 @@ func (c *child) serve() { case typeData: // If the filter role is implemented, read the data stream here. case typeAbortRequest: - requests[rec.h.Id] = nil, false + delete(requests, rec.h.Id) c.conn.writeEndRequest(rec.h.Id, 0, statusRequestComplete) if !req.keepConn { // connection will close upon return diff --git a/libgo/go/http/fcgi/fcgi_test.go b/libgo/go/http/fcgi/fcgi_test.go index 16a6243..5c8e46b 100644 --- a/libgo/go/http/fcgi/fcgi_test.go +++ b/libgo/go/http/fcgi/fcgi_test.go @@ -53,13 +53,13 @@ var streamTests = []struct { {"two records", typeStdin, 300, make([]byte, 66000), bytes.Join([][]byte{ // header for the first record - []byte{1, typeStdin, 0x01, 0x2C, 0xFF, 0xFF, 1, 0}, + {1, typeStdin, 0x01, 0x2C, 0xFF, 0xFF, 1, 0}, make([]byte, 65536), // header for the second - []byte{1, typeStdin, 0x01, 0x2C, 0x01, 0xD1, 7, 0}, + {1, typeStdin, 0x01, 0x2C, 0x01, 0xD1, 7, 0}, make([]byte, 472), // header for the empty record - []byte{1, typeStdin, 0x01, 0x2C, 0, 0, 0, 0}, + {1, typeStdin, 0x01, 0x2C, 0, 0, 0, 0}, }, nil), }, diff --git a/libgo/go/http/filetransport.go b/libgo/go/http/filetransport.go new file mode 100644 index 0000000..78f3aa2 --- /dev/null +++ b/libgo/go/http/filetransport.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "fmt" + "io" + "os" +) + +// fileTransport implements RoundTripper for the 'file' protocol. +type fileTransport struct { + fh fileHandler +} + +// NewFileTransport returns a new RoundTripper, serving the provided +// FileSystem. The returned RoundTripper ignores the URL host in its +// incoming requests, as well as most other properties of the +// request. +// +// The typical use case for NewFileTransport is to register the "file" +// protocol with a Transport, as in: +// +// t := &http.Transport{} +// t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) +// c := &http.Client{Transport: t} +// res, err := c.Get("file:///etc/passwd") +// ... +func NewFileTransport(fs FileSystem) RoundTripper { + return fileTransport{fileHandler{fs}} +} + +func (t fileTransport) RoundTrip(req *Request) (resp *Response, err os.Error) { + // We start ServeHTTP in a goroutine, which may take a long + // time if the file is large. The newPopulateResponseWriter + // call returns a channel which either ServeHTTP or finish() + // sends our *Response on, once the *Response itself has been + // populated (even if the body itself is still being + // written to the res.Body, a pipe) + rw, resc := newPopulateResponseWriter() + go func() { + t.fh.ServeHTTP(rw, req) + rw.finish() + }() + return <-resc, nil +} + +func newPopulateResponseWriter() (*populateResponse, <-chan *Response) { + pr, pw := io.Pipe() + rw := &populateResponse{ + ch: make(chan *Response), + pw: pw, + res: &Response{ + Proto: "HTTP/1.0", + ProtoMajor: 1, + Header: make(Header), + Close: true, + Body: pr, + }, + } + return rw, rw.ch +} + +// populateResponse is a ResponseWriter that populates the *Response +// in res, and writes its body to a pipe connected to the response +// body. Once writes begin or finish() is called, the response is sent +// on ch. +type populateResponse struct { + res *Response + ch chan *Response + wroteHeader bool + hasContent bool + sentResponse bool + pw *io.PipeWriter +} + +func (pr *populateResponse) finish() { + if !pr.wroteHeader { + pr.WriteHeader(500) + } + if !pr.sentResponse { + pr.sendResponse() + } + pr.pw.Close() +} + +func (pr *populateResponse) sendResponse() { + if pr.sentResponse { + return + } + pr.sentResponse = true + + if pr.hasContent { + pr.res.ContentLength = -1 + } + pr.ch <- pr.res +} + +func (pr *populateResponse) Header() Header { + return pr.res.Header +} + +func (pr *populateResponse) WriteHeader(code int) { + if pr.wroteHeader { + return + } + pr.wroteHeader = true + + pr.res.StatusCode = code + pr.res.Status = fmt.Sprintf("%d %s", code, StatusText(code)) +} + +func (pr *populateResponse) Write(p []byte) (n int, err os.Error) { + if !pr.wroteHeader { + pr.WriteHeader(StatusOK) + } + pr.hasContent = true + if !pr.sentResponse { + pr.sendResponse() + } + return pr.pw.Write(p) +} diff --git a/libgo/go/http/filetransport_test.go b/libgo/go/http/filetransport_test.go new file mode 100644 index 0000000..2634243 --- /dev/null +++ b/libgo/go/http/filetransport_test.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "http" + "io/ioutil" + "path/filepath" + "os" + "testing" +) + +func checker(t *testing.T) func(string, os.Error) { + return func(call string, err os.Error) { + if err == nil { + return + } + t.Fatalf("%s: %v", call, err) + } +} + +func TestFileTransport(t *testing.T) { + check := checker(t) + + dname, err := ioutil.TempDir("", "") + check("TempDir", err) + fname := filepath.Join(dname, "foo.txt") + err = ioutil.WriteFile(fname, []byte("Bar"), 0644) + check("WriteFile", err) + + tr := &http.Transport{} + tr.RegisterProtocol("file", http.NewFileTransport(http.Dir(dname))) + c := &http.Client{Transport: tr} + + fooURLs := []string{"file:///foo.txt", "file://../foo.txt"} + for _, urlstr := range fooURLs { + res, err := c.Get(urlstr) + check("Get "+urlstr, err) + if res.StatusCode != 200 { + t.Errorf("for %s, StatusCode = %d, want 200", urlstr, res.StatusCode) + } + if res.ContentLength != -1 { + t.Errorf("for %s, ContentLength = %d, want -1", urlstr, res.ContentLength) + } + if res.Body == nil { + t.Fatalf("for %s, nil Body", urlstr) + } + slurp, err := ioutil.ReadAll(res.Body) + check("ReadAll "+urlstr, err) + if string(slurp) != "Bar" { + t.Errorf("for %s, got content %q, want %q", urlstr, string(slurp), "Bar") + } + } + + const badURL = "file://../no-exist.txt" + res, err := c.Get(badURL) + check("Get "+badURL, err) + if res.StatusCode != 404 { + t.Errorf("for %s, StatusCode = %d, want 404", badURL, res.StatusCode) + } +} diff --git a/libgo/go/http/fs.go b/libgo/go/http/fs.go index 2c7c636..6d71665 100644 --- a/libgo/go/http/fs.go +++ b/libgo/go/http/fs.go @@ -219,7 +219,7 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec w.WriteHeader(code) if r.Method != "HEAD" { - io.Copyn(w, f, size) + io.CopyN(w, f, size) } } diff --git a/libgo/go/http/header.go b/libgo/go/http/header.go index 08b0771..aaaa92a 100644 --- a/libgo/go/http/header.go +++ b/libgo/go/http/header.go @@ -47,6 +47,8 @@ func (h Header) Write(w io.Writer) os.Error { return h.WriteSubset(w, nil) } +var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ") + // WriteSubset writes a header in wire format. // If exclude is not nil, keys where exclude[key] == true are not written. func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) os.Error { @@ -59,8 +61,7 @@ func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) os.Error { sort.Strings(keys) for _, k := range keys { for _, v := range h[k] { - v = strings.Replace(v, "\n", " ", -1) - v = strings.Replace(v, "\r", " ", -1) + v = headerNewlineToSpace.Replace(v) v = strings.TrimSpace(v) if _, err := fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { return err diff --git a/libgo/go/http/httptest/server.go b/libgo/go/http/httptest/server.go index 2ec36d0..43a48eb 100644 --- a/libgo/go/http/httptest/server.go +++ b/libgo/go/http/httptest/server.go @@ -23,6 +23,10 @@ type Server struct { URL string // base URL of form http://ipaddr:port with no trailing slash Listener net.Listener TLS *tls.Config // nil if not using using TLS + + // Config may be changed after calling NewUnstartedServer and + // before Start or StartTLS. + Config *http.Server } // historyListener keeps track of all connections that it's ever @@ -41,6 +45,13 @@ func (hs *historyListener) Accept() (c net.Conn, err os.Error) { } func newLocalListener() net.Listener { + if *serve != "" { + l, err := net.Listen("tcp", *serve) + if err != nil { + panic(fmt.Sprintf("httptest: failed to listen on %v: %v", *serve, err)) + } + return l + } l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { @@ -59,51 +70,66 @@ var serve = flag.String("httptest.serve", "", "if non-empty, httptest.NewServer // NewServer starts and returns a new Server. // The caller should call Close when finished, to shut it down. func NewServer(handler http.Handler) *Server { - ts := new(Server) - var l net.Listener - if *serve != "" { - var err os.Error - l, err = net.Listen("tcp", *serve) - if err != nil { - panic(fmt.Sprintf("httptest: failed to listen on %v: %v", *serve, err)) - } - } else { - l = newLocalListener() + ts := NewUnstartedServer(handler) + ts.Start() + return ts +} + +// NewUnstartedServer returns a new Server but doesn't start it. +// +// After changing its configuration, the caller should call Start or +// StartTLS. +// +// The caller should call Close when finished, to shut it down. +func NewUnstartedServer(handler http.Handler) *Server { + return &Server{ + Listener: newLocalListener(), + Config: &http.Server{Handler: handler}, } - ts.Listener = &historyListener{l, make([]net.Conn, 0)} - ts.URL = "http://" + l.Addr().String() - server := &http.Server{Handler: handler} - go server.Serve(ts.Listener) +} + +// Start starts a server from NewUnstartedServer. +func (s *Server) Start() { + if s.URL != "" { + panic("Server already started") + } + s.Listener = &historyListener{s.Listener, make([]net.Conn, 0)} + s.URL = "http://" + s.Listener.Addr().String() + go s.Config.Serve(s.Listener) if *serve != "" { - fmt.Println(os.Stderr, "httptest: serving on", ts.URL) + fmt.Println(os.Stderr, "httptest: serving on", s.URL) select {} } - return ts } -// NewTLSServer starts and returns a new Server using TLS. -// The caller should call Close when finished, to shut it down. -func NewTLSServer(handler http.Handler) *Server { - l := newLocalListener() - ts := new(Server) - +// StartTLS starts TLS on a server from NewUnstartedServer. +func (s *Server) StartTLS() { + if s.URL != "" { + panic("Server already started") + } cert, err := tls.X509KeyPair(localhostCert, localhostKey) if err != nil { panic(fmt.Sprintf("httptest: NewTLSServer: %v", err)) } - ts.TLS = &tls.Config{ + s.TLS = &tls.Config{ Rand: rand.Reader, Time: time.Seconds, NextProtos: []string{"http/1.1"}, Certificates: []tls.Certificate{cert}, } - tlsListener := tls.NewListener(l, ts.TLS) + tlsListener := tls.NewListener(s.Listener, s.TLS) + + s.Listener = &historyListener{tlsListener, make([]net.Conn, 0)} + s.URL = "https://" + s.Listener.Addr().String() + go s.Config.Serve(s.Listener) +} - ts.Listener = &historyListener{tlsListener, make([]net.Conn, 0)} - ts.URL = "https://" + l.Addr().String() - server := &http.Server{Handler: handler} - go server.Serve(ts.Listener) +// NewTLSServer starts and returns a new Server using TLS. +// The caller should call Close when finished, to shut it down. +func NewTLSServer(handler http.Handler) *Server { + ts := NewUnstartedServer(handler) + ts.StartTLS() return ts } diff --git a/libgo/go/http/persist.go b/libgo/go/http/persist.go index 78bf905..f73e6c6 100644 --- a/libgo/go/http/persist.go +++ b/libgo/go/http/persist.go @@ -165,7 +165,7 @@ func (sc *ServerConn) Write(req *Request, resp *Response) os.Error { // Retrieve the pipeline ID of this request/response pair sc.lk.Lock() id, ok := sc.pipereq[req] - sc.pipereq[req] = 0, false + delete(sc.pipereq, req) if !ok { sc.lk.Unlock() return ErrPipeline @@ -353,7 +353,7 @@ func (cc *ClientConn) readUsing(req *Request, readRes func(*bufio.Reader, *Reque // Retrieve the pipeline ID of this request/response pair cc.lk.Lock() id, ok := cc.pipereq[req] - cc.pipereq[req] = 0, false + delete(cc.pipereq, req) if !ok { cc.lk.Unlock() return nil, ErrPipeline diff --git a/libgo/go/http/readrequest_test.go b/libgo/go/http/readrequest_test.go index f6dc99e..6d9042a 100644 --- a/libgo/go/http/readrequest_test.go +++ b/libgo/go/http/readrequest_test.go @@ -40,7 +40,6 @@ var reqTests = []reqTest{ &Request{ Method: "GET", - RawURL: "http://www.techcrunch.com/", URL: &url.URL{ Raw: "http://www.techcrunch.com/", Scheme: "http", @@ -83,7 +82,6 @@ var reqTests = []reqTest{ &Request{ Method: "GET", - RawURL: "/", URL: &url.URL{ Raw: "/", Path: "/", @@ -110,7 +108,6 @@ var reqTests = []reqTest{ &Request{ Method: "GET", - RawURL: "//user@host/is/actually/a/path/", URL: &url.URL{ Raw: "//user@host/is/actually/a/path/", Scheme: "", diff --git a/libgo/go/http/request.go b/libgo/go/http/request.go index ed41fa4..02317e0 100644 --- a/libgo/go/http/request.go +++ b/libgo/go/http/request.go @@ -64,18 +64,24 @@ func (e *badStringError) String() string { return fmt.Sprintf("%s %q", e.what, e // Headers that Request.Write handles itself and should be skipped. var reqWriteExcludeHeader = map[string]bool{ - "Host": true, + "Host": true, // not in Header map anyway "User-Agent": true, "Content-Length": true, "Transfer-Encoding": true, "Trailer": true, } +var reqWriteExcludeHeaderDump = map[string]bool{ + "Host": true, // not in Header map anyway + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + // A Request represents a parsed HTTP request header. type Request struct { - Method string // GET, POST, PUT, etc. - RawURL string // The raw URL given in the request. - URL *url.URL // Parsed URL. + Method string // GET, POST, PUT, etc. + URL *url.URL // The protocol version for incoming requests. // Outgoing requests always use HTTP/1.1. @@ -234,8 +240,8 @@ func (r *Request) multipartReader() (*multipart.Reader, os.Error) { if v == "" { return nil, ErrNotMultipart } - d, params := mime.ParseMediaType(v) - if d != "multipart/form-data" { + d, params, err := mime.ParseMediaType(v) + if err != nil || d != "multipart/form-data" { return nil, ErrNotMultipart } boundary, ok := params["boundary"] @@ -258,7 +264,7 @@ const defaultUserAgent = "Go http package" // Write writes an HTTP/1.1 request -- header and body -- in wire format. // This method consults the following fields of req: // Host -// RawURL, if non-empty, or else URL +// URL // Method (defaults to "GET") // Header // ContentLength @@ -269,19 +275,66 @@ const defaultUserAgent = "Go http package" // hasn't been set to "identity", Write adds "Transfer-Encoding: // chunked" to the header. Body is closed after it is sent. func (req *Request) Write(w io.Writer) os.Error { - return req.write(w, false) + return req.write(w, false, nil) } // WriteProxy is like Write but writes the request in the form -// expected by an HTTP proxy. It includes the scheme and host -// name in the URI instead of using a separate Host: header line. -// If req.RawURL is non-empty, WriteProxy uses it unchanged -// instead of URL but still omits the Host: header. +// expected by an HTTP proxy. In particular, WriteProxy writes the +// initial Request-URI line of the request with an absolute URI, per +// section 5.1.2 of RFC 2616, including the scheme and host. In +// either case, WriteProxy also writes a Host header, using either +// req.Host or req.URL.Host. func (req *Request) WriteProxy(w io.Writer) os.Error { - return req.write(w, true) + return req.write(w, true, nil) +} + +func (req *Request) dumpWrite(w io.Writer) os.Error { + // TODO(bradfitz): RawPath here? + urlStr := valueOrDefault(req.URL.EncodedPath(), "/") + if req.URL.RawQuery != "" { + urlStr += "?" + req.URL.RawQuery + } + + bw := bufio.NewWriter(w) + fmt.Fprintf(bw, "%s %s HTTP/%d.%d\r\n", valueOrDefault(req.Method, "GET"), urlStr, + req.ProtoMajor, req.ProtoMinor) + + host := req.Host + if host == "" && req.URL != nil { + host = req.URL.Host + } + if host != "" { + fmt.Fprintf(bw, "Host: %s\r\n", host) + } + + // Process Body,ContentLength,Close,Trailer + tw, err := newTransferWriter(req) + if err != nil { + return err + } + err = tw.WriteHeader(bw) + if err != nil { + return err + } + + err = req.Header.WriteSubset(bw, reqWriteExcludeHeaderDump) + if err != nil { + return err + } + + io.WriteString(bw, "\r\n") + + // Write body and trailer + err = tw.WriteBody(bw) + if err != nil { + return err + } + bw.Flush() + return nil } -func (req *Request) write(w io.Writer, usingProxy bool) os.Error { +// extraHeaders may be nil +func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) os.Error { host := req.Host if host == "" { if req.URL == nil { @@ -290,9 +343,12 @@ func (req *Request) write(w io.Writer, usingProxy bool) os.Error { host = req.URL.Host } - urlStr := req.RawURL + urlStr := req.URL.RawPath + if strings.HasPrefix(urlStr, "?") { + urlStr = "/" + urlStr // Issue 2344 + } if urlStr == "" { - urlStr = valueOrDefault(req.URL.EncodedPath(), "/") + urlStr = valueOrDefault(req.URL.RawPath, valueOrDefault(req.URL.EncodedPath(), "/")) if req.URL.RawQuery != "" { urlStr += "?" + req.URL.RawQuery } @@ -303,6 +359,7 @@ func (req *Request) write(w io.Writer, usingProxy bool) os.Error { urlStr = req.URL.Scheme + "://" + host + urlStr } } + // TODO(bradfitz): escape at least newlines in urlStr? bw := bufio.NewWriter(w) fmt.Fprintf(bw, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), urlStr) @@ -338,6 +395,13 @@ func (req *Request) write(w io.Writer, usingProxy bool) os.Error { return err } + if extraHeaders != nil { + err = extraHeaders.Write(bw) + if err != nil { + return err + } + } + io.WriteString(bw, "\r\n") // Write body and trailer @@ -542,13 +606,14 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { if f = strings.SplitN(s, " ", 3); len(f) < 3 { return nil, &badStringError{"malformed HTTP request", s} } - req.Method, req.RawURL, req.Proto = f[0], f[1], f[2] + var rawurl string + req.Method, rawurl, req.Proto = f[0], f[1], f[2] var ok bool if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { return nil, &badStringError{"malformed HTTP version", req.Proto} } - if req.URL, err = url.ParseRequest(req.RawURL); err != nil { + if req.URL, err = url.ParseRequest(rawurl); err != nil { return nil, err } @@ -608,27 +673,77 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { return req, nil } -// ParseForm parses the raw query. -// For POST requests, it also parses the request body as a form. +// MaxBytesReader is similar to io.LimitReader but is intended for +// limiting the size of incoming request bodies. In contrast to +// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a +// non-EOF error for a Read beyond the limit, and Closes the +// underlying reader when its Close method is called. +// +// MaxBytesReader prevents clients from accidentally or maliciously +// sending a large request and wasting server resources. +func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { + return &maxBytesReader{w: w, r: r, n: n} +} + +type maxBytesReader struct { + w ResponseWriter + r io.ReadCloser // underlying reader + n int64 // max bytes remaining + stopped bool +} + +func (l *maxBytesReader) Read(p []byte) (n int, err os.Error) { + if l.n <= 0 { + if !l.stopped { + l.stopped = true + if res, ok := l.w.(*response); ok { + res.requestTooLarge() + } + } + return 0, os.NewError("http: request body too large") + } + if int64(len(p)) > l.n { + p = p[:l.n] + } + n, err = l.r.Read(p) + l.n -= int64(n) + return +} + +func (l *maxBytesReader) Close() os.Error { + return l.r.Close() +} + +// ParseForm parses the raw query from the URL. +// +// For POST or PUT requests, it also parses the request body as a form. +// If the request Body's size has not already been limited by MaxBytesReader, +// the size is capped at 10MB. +// // ParseMultipartForm calls ParseForm automatically. // It is idempotent. func (r *Request) ParseForm() (err os.Error) { if r.Form != nil { return } - if r.URL != nil { r.Form, err = url.ParseQuery(r.URL.RawQuery) } - if r.Method == "POST" { + if r.Method == "POST" || r.Method == "PUT" { if r.Body == nil { return os.NewError("missing form body") } ct := r.Header.Get("Content-Type") - switch strings.SplitN(ct, ";", 2)[0] { - case "text/plain", "application/x-www-form-urlencoded", "": - const maxFormSize = int64(10 << 20) // 10 MB is a lot of text. - b, e := ioutil.ReadAll(io.LimitReader(r.Body, maxFormSize+1)) + ct, _, err := mime.ParseMediaType(ct) + switch { + case ct == "text/plain" || ct == "application/x-www-form-urlencoded" || ct == "": + var reader io.Reader = r.Body + maxFormSize := int64(1<<63 - 1) + if _, ok := r.Body.(*maxBytesReader); !ok { + maxFormSize = int64(10 << 20) // 10 MB is a lot of text. + reader = io.LimitReader(r.Body, maxFormSize+1) + } + b, e := ioutil.ReadAll(reader) if e != nil { if err == nil { err = e @@ -652,8 +767,13 @@ func (r *Request) ParseForm() (err os.Error) { r.Form.Add(k, value) } } - case "multipart/form-data": - // handled by ParseMultipartForm + case ct == "multipart/form-data": + // handled by ParseMultipartForm (which is calling us, or should be) + // TODO(bradfitz): there are too many possible + // orders to call too many functions here. + // Clean this up and write more tests. + // request_test.go contains the start of this, + // in TestRequestMultipartCallOrder. default: return &badStringError{"unknown Content-Type", ct} } diff --git a/libgo/go/http/request_test.go b/libgo/go/http/request_test.go index 869cd57..175d6f1 100644 --- a/libgo/go/http/request_test.go +++ b/libgo/go/http/request_test.go @@ -20,57 +20,6 @@ import ( "url" ) -type stringMultimap map[string][]string - -type parseTest struct { - query string - out stringMultimap -} - -var parseTests = []parseTest{ - { - query: "a=1&b=2", - out: stringMultimap{"a": []string{"1"}, "b": []string{"2"}}, - }, - { - query: "a=1&a=2&a=banana", - out: stringMultimap{"a": []string{"1", "2", "banana"}}, - }, - { - query: "ascii=%3Ckey%3A+0x90%3E", - out: stringMultimap{"ascii": []string{"<key: 0x90>"}}, - }, -} - -func TestParseForm(t *testing.T) { - for i, test := range parseTests { - form, err := url.ParseQuery(test.query) - if err != nil { - t.Errorf("test %d: Unexpected error: %v", i, err) - continue - } - if len(form) != len(test.out) { - t.Errorf("test %d: len(form) = %d, want %d", i, len(form), len(test.out)) - } - for k, evs := range test.out { - vs, ok := form[k] - if !ok { - t.Errorf("test %d: Missing key %q", i, k) - continue - } - if len(vs) != len(evs) { - t.Errorf("test %d: len(form[%q]) = %d, want %d", i, k, len(vs), len(evs)) - continue - } - for j, ev := range evs { - if v := vs[j]; v != ev { - t.Errorf("test %d: form[%q][%d] = %q, want %q", i, k, j, v, ev) - } - } - } - } -} - func TestQuery(t *testing.T) { req := &Request{Method: "GET"} req.URL, _ = url.Parse("http://www.google.com/search?q=foo&q=bar") diff --git a/libgo/go/http/requestwrite_test.go b/libgo/go/http/requestwrite_test.go index 458f0bd..194f6dd 100644 --- a/libgo/go/http/requestwrite_test.go +++ b/libgo/go/http/requestwrite_test.go @@ -16,18 +16,22 @@ import ( ) type reqWriteTest struct { - Req Request - Body interface{} // optional []byte or func() io.ReadCloser to populate Req.Body - Raw string - RawProxy string + Req Request + Body interface{} // optional []byte or func() io.ReadCloser to populate Req.Body + + // Any of these three may be empty to skip that test. + WantWrite string // Request.Write + WantProxy string // Request.WriteProxy + WantDump string // DumpRequest + + WantError os.Error // wanted error from Request.Write } var reqWriteTests = []reqWriteTest{ // HTTP/1.1 => chunked coding; no body; no trailer { - Request{ + Req: Request{ Method: "GET", - RawURL: "http://www.techcrunch.com/", URL: &url.URL{ Raw: "http://www.techcrunch.com/", Scheme: "http", @@ -57,9 +61,7 @@ var reqWriteTests = []reqWriteTest{ Form: map[string][]string{}, }, - nil, - - "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + + WantWrite: "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + "Host: www.techcrunch.com\r\n" + "User-Agent: Fake\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + @@ -69,7 +71,7 @@ var reqWriteTests = []reqWriteTest{ "Keep-Alive: 300\r\n" + "Proxy-Connection: keep-alive\r\n\r\n", - "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + + WantProxy: "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + "Host: www.techcrunch.com\r\n" + "User-Agent: Fake\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + @@ -81,7 +83,7 @@ var reqWriteTests = []reqWriteTest{ }, // HTTP/1.1 => chunked coding; body; empty trailer { - Request{ + Req: Request{ Method: "GET", URL: &url.URL{ Scheme: "http", @@ -94,23 +96,28 @@ var reqWriteTests = []reqWriteTest{ TransferEncoding: []string{"chunked"}, }, - []byte("abcdef"), + Body: []byte("abcdef"), - "GET /search HTTP/1.1\r\n" + + WantWrite: "GET /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + chunk("abcdef") + chunk(""), - "GET http://www.google.com/search HTTP/1.1\r\n" + + WantProxy: "GET http://www.google.com/search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + chunk("abcdef") + chunk(""), + + WantDump: "GET /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("abcdef") + chunk(""), }, // HTTP/1.1 POST => chunked coding; body; empty trailer { - Request{ + Req: Request{ Method: "POST", URL: &url.URL{ Scheme: "http", @@ -124,16 +131,16 @@ var reqWriteTests = []reqWriteTest{ TransferEncoding: []string{"chunked"}, }, - []byte("abcdef"), + Body: []byte("abcdef"), - "POST /search HTTP/1.1\r\n" + + WantWrite: "POST /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + chunk("abcdef") + chunk(""), - "POST http://www.google.com/search HTTP/1.1\r\n" + + WantProxy: "POST http://www.google.com/search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + @@ -143,7 +150,7 @@ var reqWriteTests = []reqWriteTest{ // HTTP/1.1 POST with Content-Length, no chunking { - Request{ + Req: Request{ Method: "POST", URL: &url.URL{ Scheme: "http", @@ -157,9 +164,9 @@ var reqWriteTests = []reqWriteTest{ ContentLength: 6, }, - []byte("abcdef"), + Body: []byte("abcdef"), - "POST /search HTTP/1.1\r\n" + + WantWrite: "POST /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + @@ -167,7 +174,7 @@ var reqWriteTests = []reqWriteTest{ "\r\n" + "abcdef", - "POST http://www.google.com/search HTTP/1.1\r\n" + + WantProxy: "POST http://www.google.com/search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + @@ -178,9 +185,9 @@ var reqWriteTests = []reqWriteTest{ // HTTP/1.1 POST with Content-Length in headers { - Request{ + Req: Request{ Method: "POST", - RawURL: "http://example.com/", + URL: mustParseURL("http://example.com/"), Host: "example.com", Header: Header{ "Content-Length": []string{"10"}, // ignored @@ -188,16 +195,16 @@ var reqWriteTests = []reqWriteTest{ ContentLength: 6, }, - []byte("abcdef"), + Body: []byte("abcdef"), - "POST http://example.com/ HTTP/1.1\r\n" + + WantWrite: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go http package\r\n" + "Content-Length: 6\r\n" + "\r\n" + "abcdef", - "POST http://example.com/ HTTP/1.1\r\n" + + WantProxy: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go http package\r\n" + "Content-Length: 6\r\n" + @@ -207,21 +214,13 @@ var reqWriteTests = []reqWriteTest{ // default to HTTP/1.1 { - Request{ + Req: Request{ Method: "GET", - RawURL: "/search", + URL: mustParseURL("/search"), Host: "www.google.com", }, - nil, - - "GET /search HTTP/1.1\r\n" + - "Host: www.google.com\r\n" + - "User-Agent: Go http package\r\n" + - "\r\n", - - // Looks weird but RawURL overrides what WriteProxy would choose. - "GET /search HTTP/1.1\r\n" + + WantWrite: "GET /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "\r\n", @@ -229,53 +228,125 @@ var reqWriteTests = []reqWriteTest{ // Request with a 0 ContentLength and a 0 byte body. { - Request{ + Req: Request{ Method: "POST", - RawURL: "/", + URL: mustParseURL("/"), Host: "example.com", ProtoMajor: 1, ProtoMinor: 1, ContentLength: 0, // as if unset by user }, - func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 0)) }, + Body: func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 0)) }, - "POST / HTTP/1.1\r\n" + + // RFC 2616 Section 14.13 says Content-Length should be specified + // unless body is prohibited by the request method. + // Also, nginx expects it for POST and PUT. + WantWrite: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go http package\r\n" + + "Content-Length: 0\r\n" + "\r\n", - "POST / HTTP/1.1\r\n" + + WantProxy: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go http package\r\n" + + "Content-Length: 0\r\n" + "\r\n", }, // Request with a 0 ContentLength and a 1 byte body. { - Request{ + Req: Request{ Method: "POST", - RawURL: "/", + URL: mustParseURL("/"), Host: "example.com", ProtoMajor: 1, ProtoMinor: 1, ContentLength: 0, // as if unset by user }, - func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 1)) }, + Body: func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 1)) }, - "POST / HTTP/1.1\r\n" + + WantWrite: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + chunk("x") + chunk(""), - "POST / HTTP/1.1\r\n" + + WantProxy: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + chunk("x") + chunk(""), }, + + // Request with a ContentLength of 10 but a 5 byte body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 10, // but we're going to send only 5 bytes + }, + Body: []byte("12345"), + WantError: os.NewError("http: Request.ContentLength=10 with Body length 5"), + }, + + // Request with a ContentLength of 4 but an 8 byte body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 4, // but we're going to try to send 8 bytes + }, + Body: []byte("12345678"), + WantError: os.NewError("http: Request.ContentLength=4 with Body length 8"), + }, + + // Request with a 5 ContentLength and nil body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 5, // but we'll omit the body + }, + WantError: os.NewError("http: Request.ContentLength=5 with nil Body"), + }, + + // Verify that DumpRequest preserves the HTTP version number, doesn't add a Host, + // and doesn't add a User-Agent. + { + Req: Request{ + Method: "GET", + URL: mustParseURL("/foo"), + ProtoMajor: 1, + ProtoMinor: 0, + Header: Header{ + "X-Foo": []string{"X-Bar"}, + }, + }, + + // We can dump it: + WantDump: "GET /foo HTTP/1.0\r\n" + + "X-Foo: X-Bar\r\n\r\n", + + // .. but we can't call Request.Write on it, due to its lack of Host header. + // TODO(bradfitz): there might be an argument to allow this, but for now I'd + // rather let HTTP/1.0 continue to die. + WantWrite: "GET /foo HTTP/1.1\r\n" + + "Host: \r\n" + + "User-Agent: Go http package\r\n" + + "X-Foo: X-Bar\r\n\r\n", + }, } func TestRequestWrite(t *testing.T) { @@ -283,6 +354,9 @@ func TestRequestWrite(t *testing.T) { tt := &reqWriteTests[i] setBody := func() { + if tt.Body == nil { + return + } switch b := tt.Body.(type) { case []byte: tt.Req.Body = ioutil.NopCloser(bytes.NewBuffer(b)) @@ -290,37 +364,55 @@ func TestRequestWrite(t *testing.T) { tt.Req.Body = b() } } - if tt.Body != nil { - setBody() - } + setBody() if tt.Req.Header == nil { tt.Req.Header = make(Header) } + var braw bytes.Buffer err := tt.Req.Write(&braw) - if err != nil { - t.Errorf("error writing #%d: %s", i, err) + if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.WantError); g != e { + t.Errorf("writing #%d, err = %q, want %q", i, g, e) continue } - sraw := braw.String() - if sraw != tt.Raw { - t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, tt.Raw, sraw) + if err != nil { continue } - if tt.Body != nil { - setBody() + if tt.WantWrite != "" { + sraw := braw.String() + if sraw != tt.WantWrite { + t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantWrite, sraw) + continue + } } - var praw bytes.Buffer - err = tt.Req.WriteProxy(&praw) - if err != nil { - t.Errorf("error writing #%d: %s", i, err) - continue + + if tt.WantProxy != "" { + setBody() + var praw bytes.Buffer + err = tt.Req.WriteProxy(&praw) + if err != nil { + t.Errorf("WriteProxy #%d: %s", i, err) + continue + } + sraw := praw.String() + if sraw != tt.WantProxy { + t.Errorf("Test Proxy %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantProxy, sraw) + continue + } } - sraw = praw.String() - if sraw != tt.RawProxy { - t.Errorf("Test Proxy %d, expecting:\n%s\nGot:\n%s\n", i, tt.RawProxy, sraw) - continue + + if tt.WantDump != "" { + setBody() + dump, err := DumpRequest(&tt.Req, true) + if err != nil { + t.Errorf("DumpRequest #%d: %s", i, err) + continue + } + if string(dump) != tt.WantDump { + t.Errorf("DumpRequest %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDump, string(dump)) + continue + } } } } @@ -368,3 +460,11 @@ func TestRequestWriteClosesBody(t *testing.T) { func chunk(s string) string { return fmt.Sprintf("%x\r\n%s\r\n", len(s), s) } + +func mustParseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(fmt.Sprintf("Error parsing URL %q: %v", s, err)) + } + return u +} diff --git a/libgo/go/http/response.go b/libgo/go/http/response.go index 915327a..56c65b53 100644 --- a/libgo/go/http/response.go +++ b/libgo/go/http/response.go @@ -13,6 +13,7 @@ import ( "os" "strconv" "strings" + "url" ) var respExcludeHeader = map[string]bool{ @@ -41,6 +42,10 @@ type Response struct { Header Header // Body represents the response body. + // + // The http Client and Transport guarantee that Body is always + // non-nil, even on responses without a body or responses with + // a zero-lengthed body. Body io.ReadCloser // ContentLength records the length of the associated content. The @@ -73,6 +78,23 @@ func (r *Response) Cookies() []*Cookie { return readSetCookies(r.Header) } +var ErrNoLocation = os.NewError("http: no Location header in response") + +// Location returns the URL of the response's "Location" header, +// if present. Relative redirects are resolved relative to +// the Response's Request. ErrNoLocation is returned if no +// Location header is present. +func (r *Response) Location() (*url.URL, os.Error) { + lv := r.Header.Get("Location") + if lv == "" { + return nil, ErrNoLocation + } + if r.Request != nil && r.Request.URL != nil { + return r.Request.URL.Parse(lv) + } + return url.Parse(lv) +} + // ReadResponse reads and returns an HTTP response from r. The // req parameter specifies the Request that corresponds to // this Response. Clients must call resp.Body.Close when finished diff --git a/libgo/go/http/response_test.go b/libgo/go/http/response_test.go index 1d4a234..86494bf 100644 --- a/libgo/go/http/response_test.go +++ b/libgo/go/http/response_test.go @@ -15,6 +15,7 @@ import ( "io/ioutil" "reflect" "testing" + "url" ) type respTest struct { @@ -395,3 +396,52 @@ func diff(t *testing.T, prefix string, have, want interface{}) { } } } + +type responseLocationTest struct { + location string // Response's Location header or "" + requrl string // Response.Request.URL or "" + want string + wantErr os.Error +} + +var responseLocationTests = []responseLocationTest{ + {"/foo", "http://bar.com/baz", "http://bar.com/foo", nil}, + {"http://foo.com/", "http://bar.com/baz", "http://foo.com/", nil}, + {"", "http://bar.com/baz", "", ErrNoLocation}, +} + +func TestLocationResponse(t *testing.T) { + for i, tt := range responseLocationTests { + res := new(Response) + res.Header = make(Header) + res.Header.Set("Location", tt.location) + if tt.requrl != "" { + res.Request = &Request{} + var err os.Error + res.Request.URL, err = url.Parse(tt.requrl) + if err != nil { + t.Fatalf("bad test URL %q: %v", tt.requrl, err) + } + } + + got, err := res.Location() + if tt.wantErr != nil { + if err == nil { + t.Errorf("%d. err=nil; want %q", i, tt.wantErr) + continue + } + if g, e := err.String(), tt.wantErr.String(); g != e { + t.Errorf("%d. err=%q; want %q", i, g, e) + continue + } + continue + } + if err != nil { + t.Errorf("%d. err=%q", i, err) + continue + } + if g, e := got.String(), tt.want; g != e { + t.Errorf("%d. Location=%q; want %q", i, g, e) + } + } +} diff --git a/libgo/go/http/serve_test.go b/libgo/go/http/serve_test.go index ac04033..2ff66d5 100644 --- a/libgo/go/http/serve_test.go +++ b/libgo/go/http/serve_test.go @@ -9,14 +9,15 @@ package http_test import ( "bufio" "bytes" + "crypto/tls" "fmt" . "http" "http/httptest" "io" "io/ioutil" "log" - "os" "net" + "os" "reflect" "strings" "syscall" @@ -356,18 +357,17 @@ func TestIdentityResponse(t *testing.T) { if err != nil { t.Fatalf("error writing: %v", err) } - // The next ReadAll will hang for a failing test, so use a Timer instead - // to fail more traditionally - timer := time.AfterFunc(2e9, func() { - t.Fatalf("Timeout expired in ReadAll.") + + // The ReadAll will hang for a failing test, so use a Timer to + // fail explicitly. + goTimeout(t, 2e9, func() { + got, _ := ioutil.ReadAll(conn) + expectedSuffix := "\r\n\r\ntoo short" + if !strings.HasSuffix(string(got), expectedSuffix) { + t.Errorf("Expected output to end with %q; got response body %q", + expectedSuffix, string(got)) + } }) - defer timer.Stop() - got, _ := ioutil.ReadAll(conn) - expectedSuffix := "\r\n\r\ntoo short" - if !strings.HasSuffix(string(got), expectedSuffix) { - t.Fatalf("Expected output to end with %q; got response body %q", - expectedSuffix, string(got)) - } } func testTcpConnectionCloses(t *testing.T, req string, h Handler) { @@ -535,6 +535,25 @@ func TestHeadResponses(t *testing.T) { } } +func TestTLSHandshakeTimeout(t *testing.T) { + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + ts.Config.ReadTimeout = 250e6 + ts.StartTLS() + defer ts.Close() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer conn.Close() + goTimeout(t, 10e9, func() { + var buf [1]byte + n, err := conn.Read(buf[:]) + if err == nil || n != 0 { + t.Errorf("Read = %d, %v; want an error and no bytes", n, err) + } + }) +} + func TestTLSServer(t *testing.T) { ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { if r.TLS != nil { @@ -545,23 +564,46 @@ func TestTLSServer(t *testing.T) { } })) defer ts.Close() - if !strings.HasPrefix(ts.URL, "https://") { - t.Fatalf("expected test TLS server to start with https://, got %q", ts.URL) - } - res, err := Get(ts.URL) + + // Connect an idle TCP connection to this server before we run + // our real tests. This idle connection used to block forever + // in the TLS handshake, preventing future connections from + // being accepted. It may prevent future accidental blocking + // in newConn. + idleConn, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { - t.Fatal(err) - } - if res == nil { - t.Fatalf("got nil Response") - } - defer res.Body.Close() - if res.Header.Get("X-TLS-Set") != "true" { - t.Errorf("expected X-TLS-Set response header") - } - if res.Header.Get("X-TLS-HandshakeComplete") != "true" { - t.Errorf("expected X-TLS-HandshakeComplete header") + t.Fatalf("Dial: %v", err) } + defer idleConn.Close() + goTimeout(t, 10e9, func() { + if !strings.HasPrefix(ts.URL, "https://") { + t.Errorf("expected test TLS server to start with https://, got %q", ts.URL) + return + } + noVerifyTransport := &Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + client := &Client{Transport: noVerifyTransport} + res, err := client.Get(ts.URL) + if err != nil { + t.Error(err) + return + } + if res == nil { + t.Errorf("got nil Response") + return + } + defer res.Body.Close() + if res.Header.Get("X-TLS-Set") != "true" { + t.Errorf("expected X-TLS-Set response header") + return + } + if res.Header.Get("X-TLS-HandshakeComplete") != "true" { + t.Errorf("expected X-TLS-HandshakeComplete header") + } + }) } type serverExpectTest struct { @@ -646,9 +688,11 @@ func TestServerExpect(t *testing.T) { } } -func TestServerConsumesRequestBody(t *testing.T) { +// Under a ~256KB (maxPostHandlerReadBytes) threshold, the server +// should consume client request bodies that a handler didn't read. +func TestServerUnreadRequestBodyLittle(t *testing.T) { conn := new(testConn) - body := strings.Repeat("x", 1<<20) + body := strings.Repeat("x", 100<<10) conn.readBuf.Write([]byte(fmt.Sprintf( "POST / HTTP/1.1\r\n"+ "Host: test\r\n"+ @@ -660,14 +704,49 @@ func TestServerConsumesRequestBody(t *testing.T) { ls := &oneConnListener{conn} go Serve(ls, HandlerFunc(func(rw ResponseWriter, req *Request) { + defer close(done) if conn.readBuf.Len() < len(body)/2 { - t.Errorf("on request, read buffer length is %d; expected about 1MB", conn.readBuf.Len()) + t.Errorf("on request, read buffer length is %d; expected about 100 KB", conn.readBuf.Len()) } rw.WriteHeader(200) if g, e := conn.readBuf.Len(), 0; g != e { t.Errorf("after WriteHeader, read buffer length is %d; want %d", g, e) } - done <- true + if c := rw.Header().Get("Connection"); c != "" { + t.Errorf(`Connection header = %q; want ""`, c) + } + })) + <-done +} + +// Over a ~256KB (maxPostHandlerReadBytes) threshold, the server +// should ignore client request bodies that a handler didn't read +// and close the connection. +func TestServerUnreadRequestBodyLarge(t *testing.T) { + conn := new(testConn) + body := strings.Repeat("x", 1<<20) + conn.readBuf.Write([]byte(fmt.Sprintf( + "POST / HTTP/1.1\r\n"+ + "Host: test\r\n"+ + "Content-Length: %d\r\n"+ + "\r\n", len(body)))) + conn.readBuf.Write([]byte(body)) + + done := make(chan bool) + + ls := &oneConnListener{conn} + go Serve(ls, HandlerFunc(func(rw ResponseWriter, req *Request) { + defer close(done) + if conn.readBuf.Len() < len(body)/2 { + t.Errorf("on request, read buffer length is %d; expected about 1MB", conn.readBuf.Len()) + } + rw.WriteHeader(200) + if conn.readBuf.Len() < len(body)/2 { + t.Errorf("post-WriteHeader, read buffer length is %d; expected about 1MB", conn.readBuf.Len()) + } + if c := rw.Header().Get("Connection"); c != "close" { + t.Errorf(`Connection header = %q; want "close"`, c) + } })) <-done } @@ -785,6 +864,14 @@ func TestZeroLengthPostAndResponse(t *testing.T) { } func TestHandlerPanic(t *testing.T) { + testHandlerPanic(t, false) +} + +func TestHandlerPanicWithHijack(t *testing.T) { + testHandlerPanic(t, true) +} + +func testHandlerPanic(t *testing.T, withHijack bool) { // Unlike the other tests that set the log output to ioutil.Discard // to quiet the output, this test uses a pipe. The pipe serves three // purposes: @@ -805,7 +892,14 @@ func TestHandlerPanic(t *testing.T) { log.SetOutput(pw) defer log.SetOutput(os.Stderr) - ts := httptest.NewServer(HandlerFunc(func(ResponseWriter, *Request) { + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if withHijack { + rwc, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Logf("unexpected error: %v", err) + } + defer rwc.Close() + } panic("intentional death for testing") })) defer ts.Close() @@ -891,9 +985,110 @@ func TestRequestLimit(t *testing.T) { // we do support it (at least currently), so we expect a response below. t.Fatalf("Do: %v", err) } - if res.StatusCode != 400 { - t.Fatalf("expected 400 response status; got: %d %s", res.StatusCode, res.Status) + if res.StatusCode != 413 { + t.Fatalf("expected 413 response status; got: %d %s", res.StatusCode, res.Status) + } +} + +type neverEnding byte + +func (b neverEnding) Read(p []byte) (n int, err os.Error) { + for i := range p { + p[i] = byte(b) + } + return len(p), nil +} + +type countReader struct { + r io.Reader + n *int64 +} + +func (cr countReader) Read(p []byte) (n int, err os.Error) { + n, err = cr.r.Read(p) + *cr.n += int64(n) + return +} + +func TestRequestBodyLimit(t *testing.T) { + const limit = 1 << 20 + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + r.Body = MaxBytesReader(w, r.Body, limit) + n, err := io.Copy(ioutil.Discard, r.Body) + if err == nil { + t.Errorf("expected error from io.Copy") + } + if n != limit { + t.Errorf("io.Copy = %d, want %d", n, limit) + } + })) + defer ts.Close() + + nWritten := int64(0) + req, _ := NewRequest("POST", ts.URL, io.LimitReader(countReader{neverEnding('a'), &nWritten}, limit*200)) + + // Send the POST, but don't care it succeeds or not. The + // remote side is going to reply and then close the TCP + // connection, and HTTP doesn't really define if that's + // allowed or not. Some HTTP clients will get the response + // and some (like ours, currently) will complain that the + // request write failed, without reading the response. + // + // But that's okay, since what we're really testing is that + // the remote side hung up on us before we wrote too much. + _, _ = DefaultClient.Do(req) + + if nWritten > limit*100 { + t.Errorf("handler restricted the request body to %d bytes, but client managed to write %d", + limit, nWritten) + } +} + +// TestClientWriteShutdown tests that if the client shuts down the write +// side of their TCP connection, the server doesn't send a 400 Bad Request. +func TestClientWriteShutdown(t *testing.T) { + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + defer ts.Close() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) } + err = conn.(*net.TCPConn).CloseWrite() + if err != nil { + t.Fatalf("Dial: %v", err) + } + donec := make(chan bool) + go func() { + defer close(donec) + bs, err := ioutil.ReadAll(conn) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + got := string(bs) + if got != "" { + t.Errorf("read %q from server; want nothing", got) + } + }() + select { + case <-donec: + case <-time.After(10e9): + t.Fatalf("timeout") + } +} + +// goTimeout runs f, failing t if f takes more than ns to complete. +func goTimeout(t *testing.T, ns int64, f func()) { + ch := make(chan bool, 2) + timer := time.AfterFunc(ns, func() { + t.Errorf("Timeout expired after %d ns", ns) + ch <- true + }) + defer timer.Stop() + go func() { + defer func() { ch <- true }() + f() + }() + <-ch } type errorListener struct { diff --git a/libgo/go/http/server.go b/libgo/go/http/server.go index b634e27..9792c60 100644 --- a/libgo/go/http/server.go +++ b/libgo/go/http/server.go @@ -16,6 +16,7 @@ import ( "crypto/tls" "fmt" "io" + "io/ioutil" "log" "net" "os" @@ -122,27 +123,46 @@ type response struct { // "Connection: keep-alive" response header and a // Content-Length. closeAfterReply bool + + // requestBodyLimitHit is set by requestTooLarge when + // maxBytesReader hits its max size. It is checked in + // WriteHeader, to make sure we don't consume the the + // remaining request body to try to advance to the next HTTP + // request. Instead, when this is set, we stop doing + // subsequent requests on this connection and stop reading + // input from it. + requestBodyLimitHit bool +} + +// requestTooLarge is called by maxBytesReader when too much input has +// been read from the client. +func (w *response) requestTooLarge() { + w.closeAfterReply = true + w.requestBodyLimitHit = true + if !w.wroteHeader { + w.Header().Set("Connection", "close") + } } type writerOnly struct { io.Writer } -func (r *response) ReadFrom(src io.Reader) (n int64, err os.Error) { - // Flush before checking r.chunking, as Flush will call +func (w *response) ReadFrom(src io.Reader) (n int64, err os.Error) { + // Flush before checking w.chunking, as Flush will call // WriteHeader if it hasn't been called yet, and WriteHeader - // is what sets r.chunking. - r.Flush() - if !r.chunking && r.bodyAllowed() && !r.needSniff { - if rf, ok := r.conn.rwc.(io.ReaderFrom); ok { + // is what sets w.chunking. + w.Flush() + if !w.chunking && w.bodyAllowed() && !w.needSniff { + if rf, ok := w.conn.rwc.(io.ReaderFrom); ok { n, err = rf.ReadFrom(src) - r.written += n + w.written += n return } } // Fall back to default io.Copy implementation. - // Use wrapper to hide r.ReadFrom from io.Copy. - return io.Copy(writerOnly{r}, src) + // Use wrapper to hide w.ReadFrom from io.Copy. + return io.Copy(writerOnly{w}, src) } // noLimit is an effective infinite upper bound for io.LimitedReader @@ -159,13 +179,6 @@ func (srv *Server) newConn(rwc net.Conn) (c *conn, err os.Error) { br := bufio.NewReader(c.lr) bw := bufio.NewWriter(rwc) c.buf = bufio.NewReadWriter(br, bw) - - if tlsConn, ok := rwc.(*tls.Conn); ok { - tlsConn.Handshake() - c.tlsState = new(tls.ConnectionState) - *c.tlsState = tlsConn.ConnectionState() - } - return c, nil } @@ -245,6 +258,17 @@ func (w *response) Header() Header { return w.header } +// maxPostHandlerReadBytes is the max number of Request.Body bytes not +// consumed by a handler that the server will read from the a client +// in order to keep a connection alive. If there are more bytes than +// this then the server to be paranoid instead sends a "Connection: +// close" response. +// +// This number is approximately what a typical machine's TCP buffer +// size is anyway. (if we have the bytes on the machine, we might as +// well read them) +const maxPostHandlerReadBytes = 256 << 10 + func (w *response) WriteHeader(code int) { if w.conn.hijacked { log.Print("http: response.WriteHeader on hijacked connection") @@ -254,18 +278,54 @@ func (w *response) WriteHeader(code int) { log.Print("http: multiple response.WriteHeader calls") return } + w.wroteHeader = true + w.status = code + + // Check for a explicit (and valid) Content-Length header. + var hasCL bool + var contentLength int64 + if clenStr := w.header.Get("Content-Length"); clenStr != "" { + var err os.Error + contentLength, err = strconv.Atoi64(clenStr) + if err == nil { + hasCL = true + } else { + log.Printf("http: invalid Content-Length of %q sent", clenStr) + w.header.Del("Content-Length") + } + } + + if w.req.wantsHttp10KeepAlive() && (w.req.Method == "HEAD" || hasCL) { + _, connectionHeaderSet := w.header["Connection"] + if !connectionHeaderSet { + w.header.Set("Connection", "keep-alive") + } + } else if !w.req.ProtoAtLeast(1, 1) { + // Client did not ask to keep connection alive. + w.closeAfterReply = true + } + + if w.header.Get("Connection") == "close" { + w.closeAfterReply = true + } // Per RFC 2616, we should consume the request body before - // replying, if the handler hasn't already done so. - if w.req.ContentLength != 0 { + // replying, if the handler hasn't already done so. But we + // don't want to do an unbounded amount of reading here for + // DoS reasons, so we only try up to a threshold. + if w.req.ContentLength != 0 && !w.closeAfterReply { ecr, isExpecter := w.req.Body.(*expectContinueReader) if !isExpecter || ecr.resp.wroteContinue { - w.req.Body.Close() + n, _ := io.CopyN(ioutil.Discard, w.req.Body, maxPostHandlerReadBytes+1) + if n >= maxPostHandlerReadBytes { + w.requestTooLarge() + w.header.Set("Connection", "close") + } else { + w.req.Body.Close() + } } } - w.wroteHeader = true - w.status = code if code == StatusNotModified { // Must not have body. for _, header := range []string{"Content-Type", "Content-Length", "Transfer-Encoding"} { @@ -288,20 +348,6 @@ func (w *response) WriteHeader(code int) { w.Header().Set("Date", time.UTC().Format(TimeFormat)) } - // Check for a explicit (and valid) Content-Length header. - var hasCL bool - var contentLength int64 - if clenStr := w.header.Get("Content-Length"); clenStr != "" { - var err os.Error - contentLength, err = strconv.Atoi64(clenStr) - if err == nil { - hasCL = true - } else { - log.Printf("http: invalid Content-Length of %q sent", clenStr) - w.header.Del("Content-Length") - } - } - te := w.header.Get("Transfer-Encoding") hasTE := te != "" if hasCL && hasTE && te != "identity" { @@ -334,20 +380,6 @@ func (w *response) WriteHeader(code int) { w.header.Del("Transfer-Encoding") // in case already set } - if w.req.wantsHttp10KeepAlive() && (w.req.Method == "HEAD" || hasCL) { - _, connectionHeaderSet := w.header["Connection"] - if !connectionHeaderSet { - w.header.Set("Connection", "keep-alive") - } - } else if !w.req.ProtoAtLeast(1, 1) { - // Client did not ask to keep connection alive. - w.closeAfterReply = true - } - - if w.header.Get("Connection") == "close" { - w.closeAfterReply = true - } - // Cannot use Content-Length with non-identity Transfer-Encoding. if w.chunking { w.header.Del("Content-Length") @@ -472,55 +504,6 @@ func (w *response) Write(data []byte) (n int, err os.Error) { return m + n, err } -// If this is an error reply (4xx or 5xx) -// and the handler wrote some data explaining the error, -// some browsers (i.e., Chrome, Internet Explorer) -// will show their own error instead unless the error is -// long enough. The minimum lengths used in those -// browsers are in the 256-512 range. -// Pad to 1024 bytes. -func errorKludge(w *response) { - const min = 1024 - - // Is this an error? - if kind := w.status / 100; kind != 4 && kind != 5 { - return - } - - // Did the handler supply any info? Enough? - if w.written == 0 || w.written >= min { - return - } - - // Is it a broken browser? - var msg string - switch agent := w.req.UserAgent(); { - case strings.Contains(agent, "MSIE"): - msg = "Internet Explorer" - case strings.Contains(agent, "Chrome/"): - msg = "Chrome" - default: - return - } - msg += " would ignore this error page if this text weren't here.\n" - - // Is it text? ("Content-Type" is always in the map) - baseType := strings.SplitN(w.header.Get("Content-Type"), ";", 2)[0] - switch baseType { - case "text/html": - io.WriteString(w, "<!-- ") - for w.written < min { - io.WriteString(w, msg) - } - io.WriteString(w, " -->") - case "text/plain": - io.WriteString(w, "\n") - for w.written < min { - io.WriteString(w, msg) - } - } -} - func (w *response) finishRequest() { // If this was an HTTP/1.0 request with keep-alive and we sent a Content-Length // back, we can make this a keep-alive response ... @@ -536,14 +519,17 @@ func (w *response) finishRequest() { if w.needSniff { w.sniff() } - errorKludge(w) if w.chunking { io.WriteString(w.conn.buf, "0\r\n") // trailer key/value pairs, followed by blank line io.WriteString(w.conn.buf, "\r\n") } w.conn.buf.Flush() - w.req.Body.Close() + // Close the body, unless we're about to close the whole TCP connection + // anyway. + if !w.closeAfterReply { + w.req.Body.Close() + } if w.req.MultipartForm != nil { w.req.MultipartForm.RemoveAll() } @@ -581,7 +567,9 @@ func (c *conn) serve() { if err == nil { return } - c.rwc.Close() + if c.rwc != nil { // may be nil if connection hijacked + c.rwc.Close() + } var buf bytes.Buffer fmt.Fprintf(&buf, "http: panic serving %v: %v\n", c.remoteAddr, err) @@ -589,17 +577,32 @@ func (c *conn) serve() { log.Print(buf.String()) }() + if tlsConn, ok := c.rwc.(*tls.Conn); ok { + if err := tlsConn.Handshake(); err != nil { + c.close() + return + } + c.tlsState = new(tls.ConnectionState) + *c.tlsState = tlsConn.ConnectionState() + } + for { w, err := c.readRequest() if err != nil { + msg := "400 Bad Request" if err == errTooLarge { // Their HTTP client may or may not be // able to read this if we're // responding to them and hanging up // while they're still writing their // request. Undefined behavior. - fmt.Fprintf(c.rwc, "HTTP/1.1 400 Request Too Large\r\n\r\n") + msg = "413 Request Entity Too Large" + } else if err == io.ErrUnexpectedEOF { + break // Don't reply + } else if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + break // Don't reply } + fmt.Fprintf(c.rwc, "HTTP/1.1 %s\r\n\r\n", msg) break } @@ -774,13 +777,16 @@ func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { } } +var htmlReplacer = strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + `"`, """, + "'", "'", +) + func htmlEscape(s string) string { - s = strings.Replace(s, "&", "&", -1) - s = strings.Replace(s, "<", "<", -1) - s = strings.Replace(s, ">", ">", -1) - s = strings.Replace(s, "\"", """, -1) - s = strings.Replace(s, "'", "'", -1) - return s + return htmlReplacer.Replace(s) } // Redirect to a fixed URL diff --git a/libgo/go/http/transfer.go b/libgo/go/http/transfer.go index b65d99a..868a114 100644 --- a/libgo/go/http/transfer.go +++ b/libgo/go/http/transfer.go @@ -7,6 +7,7 @@ package http import ( "bytes" "bufio" + "fmt" "io" "io/ioutil" "os" @@ -18,10 +19,11 @@ import ( // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. type transferWriter struct { + Method string Body io.Reader BodyCloser io.Closer ResponseToHEAD bool - ContentLength int64 + ContentLength int64 // -1 means unknown, 0 means exactly none Close bool TransferEncoding []string Trailer Header @@ -34,6 +36,10 @@ func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) { atLeastHTTP11 := false switch rr := r.(type) { case *Request: + if rr.ContentLength != 0 && rr.Body == nil { + return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) + } + t.Method = rr.Method t.Body = rr.Body t.BodyCloser = rr.Body t.ContentLength = rr.ContentLength @@ -64,6 +70,7 @@ func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) { } } case *Response: + t.Method = rr.Request.Method t.Body = rr.Body t.BodyCloser = rr.Body t.ContentLength = rr.ContentLength @@ -105,6 +112,27 @@ func noBodyExpected(requestMethod string) bool { return requestMethod == "HEAD" } +func (t *transferWriter) shouldSendContentLength() bool { + if chunked(t.TransferEncoding) { + return false + } + if t.ContentLength > 0 { + return true + } + if t.ResponseToHEAD { + return true + } + // Many servers expect a Content-Length for these methods + if t.Method == "POST" || t.Method == "PUT" { + return true + } + if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { + return true + } + + return false +} + func (t *transferWriter) WriteHeader(w io.Writer) (err os.Error) { if t.Close { _, err = io.WriteString(w, "Connection: close\r\n") @@ -116,14 +144,14 @@ func (t *transferWriter) WriteHeader(w io.Writer) (err os.Error) { // Write Content-Length and/or Transfer-Encoding whose values are a // function of the sanitized field triple (Body, ContentLength, // TransferEncoding) - if chunked(t.TransferEncoding) { - _, err = io.WriteString(w, "Transfer-Encoding: chunked\r\n") + if t.shouldSendContentLength() { + io.WriteString(w, "Content-Length: ") + _, err = io.WriteString(w, strconv.Itoa64(t.ContentLength)+"\r\n") if err != nil { return } - } else if t.ContentLength > 0 || t.ResponseToHEAD || (t.ContentLength == 0 && isIdentity(t.TransferEncoding)) { - io.WriteString(w, "Content-Length: ") - _, err = io.WriteString(w, strconv.Itoa64(t.ContentLength)+"\r\n") + } else if chunked(t.TransferEncoding) { + _, err = io.WriteString(w, "Transfer-Encoding: chunked\r\n") if err != nil { return } @@ -154,6 +182,8 @@ func (t *transferWriter) WriteHeader(w io.Writer) (err os.Error) { } func (t *transferWriter) WriteBody(w io.Writer) (err os.Error) { + var ncopy int64 + // Write body if t.Body != nil { if chunked(t.TransferEncoding) { @@ -163,9 +193,14 @@ func (t *transferWriter) WriteBody(w io.Writer) (err os.Error) { err = cw.Close() } } else if t.ContentLength == -1 { - _, err = io.Copy(w, t.Body) + ncopy, err = io.Copy(w, t.Body) } else { - _, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength)) + ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength)) + nextra, err := io.Copy(ioutil.Discard, t.Body) + if err != nil { + return err + } + ncopy += nextra } if err != nil { return err @@ -175,6 +210,11 @@ func (t *transferWriter) WriteBody(w io.Writer) (err os.Error) { } } + if t.ContentLength != -1 && t.ContentLength != ncopy { + return fmt.Errorf("http: Request.ContentLength=%d with Body length %d", + t.ContentLength, ncopy) + } + // TODO(petar): Place trailer writer code here. if chunked(t.TransferEncoding) { // Last chunk, empty trailer @@ -326,7 +366,7 @@ func fixTransferEncoding(requestMethod string, header Header) ([]string, os.Erro return nil, nil } - header["Transfer-Encoding"] = nil, false + delete(header, "Transfer-Encoding") // Head responses have no bodies, so the transfer encoding // should be ignored. @@ -359,7 +399,7 @@ func fixTransferEncoding(requestMethod string, header Header) ([]string, os.Erro // Chunked encoding trumps Content-Length. See RFC 2616 // Section 4.4. Currently len(te) > 0 implies chunked // encoding. - header["Content-Length"] = nil, false + delete(header, "Content-Length") return te, nil } @@ -478,6 +518,8 @@ type body struct { r *bufio.Reader // underlying wire-format reader for the trailer closing bool // is the connection to be closed after reading body? closed bool + + res *response // response writer for server requests, else nil } // ErrBodyReadAfterClose is returned when reading a Request Body after @@ -506,6 +548,15 @@ func (b *body) Close() os.Error { return nil } + // In a server request, don't continue reading from the client + // if we've already hit the maximum body size set by the + // handler. If this is set, that also means the TCP connection + // is about to be closed, so getting to the next HTTP request + // in the stream is not necessary. + if b.res != nil && b.res.requestBodyLimitHit { + return nil + } + if _, err := io.Copy(ioutil.Discard, b); err != nil { return err } diff --git a/libgo/go/http/transport.go b/libgo/go/http/transport.go index 4302ffa..0914af7 100644 --- a/libgo/go/http/transport.go +++ b/libgo/go/http/transport.go @@ -54,6 +54,10 @@ type Transport struct { // If Dial is nil, net.Dial is used. Dial func(net, addr string) (c net.Conn, err os.Error) + // TLSClientConfig specifies the TLS configuration to use with + // tls.Client. If nil, the default configuration is used. + TLSClientConfig *tls.Config + DisableKeepAlives bool DisableCompression bool @@ -96,12 +100,27 @@ func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, os.Error) { } } +// transportRequest is a wrapper around a *Request that adds +// optional extra headers to write. +type transportRequest struct { + *Request // original request, not to be mutated + extra Header // extra headers to write, or nil +} + +func (tr *transportRequest) extraHeaders() Header { + if tr.extra == nil { + tr.extra = make(Header) + } + return tr.extra +} + // RoundTrip implements the RoundTripper interface. func (t *Transport) RoundTrip(req *Request) (resp *Response, err os.Error) { if req.URL == nil { - if req.URL, err = url.Parse(req.RawURL); err != nil { - return - } + return nil, os.NewError("http: nil Request.URL") + } + if req.Header == nil { + return nil, os.NewError("http: nil Request.Header") } if req.URL.Scheme != "http" && req.URL.Scheme != "https" { t.lk.Lock() @@ -115,8 +134,8 @@ func (t *Transport) RoundTrip(req *Request) (resp *Response, err os.Error) { } return rt.RoundTrip(req) } - - cm, err := t.connectMethodForRequest(req) + treq := &transportRequest{Request: req} + cm, err := t.connectMethodForRequest(treq) if err != nil { return nil, err } @@ -130,7 +149,7 @@ func (t *Transport) RoundTrip(req *Request) (resp *Response, err os.Error) { return nil, err } - return pconn.roundTrip(req) + return pconn.roundTrip(treq) } // RegisterProtocol registers a new protocol with scheme. @@ -183,14 +202,14 @@ func getenvEitherCase(k string) string { return os.Getenv(strings.ToLower(k)) } -func (t *Transport) connectMethodForRequest(req *Request) (*connectMethod, os.Error) { +func (t *Transport) connectMethodForRequest(treq *transportRequest) (*connectMethod, os.Error) { cm := &connectMethod{ - targetScheme: req.URL.Scheme, - targetAddr: canonicalAddr(req.URL), + targetScheme: treq.URL.Scheme, + targetAddr: canonicalAddr(treq.URL), } if t.Proxy != nil { var err os.Error - cm.proxyURL, err = t.Proxy(req) + cm.proxyURL, err = t.Proxy(treq.Request) if err != nil { return nil, err } @@ -247,7 +266,7 @@ func (t *Transport) getIdleConn(cm *connectMethod) (pconn *persistConn) { } if len(pconns) == 1 { pconn = pconns[0] - t.idleConn[key] = nil, false + delete(t.idleConn, key) } else { // 2 or more cached connections; pop last // TODO: queue? @@ -293,25 +312,21 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { conn: conn, reqch: make(chan requestAndChan, 50), } - newClientConnFunc := NewClientConn switch { case cm.proxyURL == nil: // Do nothing. case cm.targetScheme == "http": - newClientConnFunc = NewProxyClientConn + pconn.isProxy = true if pa != "" { - pconn.mutateRequestFunc = func(req *Request) { - if req.Header == nil { - req.Header = make(Header) - } - req.Header.Set("Proxy-Authorization", pa) + pconn.mutateHeaderFunc = func(h Header) { + h.Set("Proxy-Authorization", pa) } } case cm.targetScheme == "https": connectReq := &Request{ Method: "CONNECT", - RawURL: cm.targetAddr, + URL: &url.URL{RawPath: cm.targetAddr}, Host: cm.targetAddr, Header: make(Header), } @@ -338,7 +353,7 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { if cm.targetScheme == "https" { // Initiate TLS and check remote host name against certificate. - conn = tls.Client(conn, nil) + conn = tls.Client(conn, t.TLSClientConfig) if err = conn.(*tls.Conn).Handshake(); err != nil { return nil, err } @@ -349,7 +364,7 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { } pconn.br = bufio.NewReader(pconn.conn) - pconn.cc = newClientConnFunc(conn, pconn.br) + pconn.cc = NewClientConn(conn, pconn.br) go pconn.readLoop() return pconn, nil } @@ -445,30 +460,21 @@ func (cm *connectMethod) tlsHost() string { return h } -type readResult struct { - res *Response // either res or err will be set - err os.Error -} - -type writeRequest struct { - // Set by client (in pc.roundTrip) - req *Request - resch chan *readResult - - // Set by writeLoop if an error writing headers. - writeErr os.Error -} - // persistConn wraps a connection, usually a persistent one // (but may be used for non-keep-alive requests as well) type persistConn struct { - t *Transport - cacheKey string // its connectMethod.String() - conn net.Conn - cc *ClientConn - br *bufio.Reader - reqch chan requestAndChan // written by roundTrip(); read by readLoop() - mutateRequestFunc func(*Request) // nil or func to modify each outbound request + t *Transport + cacheKey string // its connectMethod.String() + conn net.Conn + cc *ClientConn + br *bufio.Reader + reqch chan requestAndChan // written by roundTrip(); read by readLoop() + isProxy bool + + // mutateHeaderFunc is an optional func to modify extra + // headers on each outbound request before it's written. (the + // original Request given to RoundTrip is not modified) + mutateHeaderFunc func(Header) lk sync.Mutex // guards numExpectedResponses and broken numExpectedResponses int @@ -487,12 +493,24 @@ func (pc *persistConn) expectingResponse() bool { return pc.numExpectedResponses > 0 } +var remoteSideClosedFunc func(os.Error) bool // or nil to use default + +func remoteSideClosed(err os.Error) bool { + if err == os.EOF || err == os.EINVAL { + return true + } + if remoteSideClosedFunc != nil { + return remoteSideClosedFunc(err) + } + return false +} + func (pc *persistConn) readLoop() { alive := true for alive { pb, err := pc.br.Peek(1) if err != nil { - if (err == os.EOF || err == os.EINVAL) && !pc.expectingResponse() { + if remoteSideClosed(err) && !pc.expectingResponse() { // Remote side closed on us. (We probably hit their // max idle timeout) pc.close() @@ -512,9 +530,6 @@ func (pc *persistConn) readLoop() { if err != nil || resp.ContentLength == 0 { return resp, err } - if rc.addedGzip { - forReq.Header.Del("Accept-Encoding") - } if rc.addedGzip && resp.Header.Get("Content-Encoding") == "gzip" { resp.Header.Del("Content-Encoding") resp.Header.Del("Content-Length") @@ -590,9 +605,9 @@ type requestAndChan struct { addedGzip bool } -func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) { - if pc.mutateRequestFunc != nil { - pc.mutateRequestFunc(req) +func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err os.Error) { + if pc.mutateHeaderFunc != nil { + pc.mutateHeaderFunc(req.extraHeaders()) } // Ask for a compressed version if the caller didn't set their @@ -602,24 +617,28 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) { requestedGzip := false if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" { // Request gzip only, not deflate. Deflate is ambiguous and - // as universally supported anyway. + // not as universally supported anyway. // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 requestedGzip = true - req.Header.Set("Accept-Encoding", "gzip") + req.extraHeaders().Set("Accept-Encoding", "gzip") } pc.lk.Lock() pc.numExpectedResponses++ pc.lk.Unlock() - err = pc.cc.Write(req) + pc.cc.writeReq = func(r *Request, w io.Writer) os.Error { + return r.write(w, pc.isProxy, req.extra) + } + + err = pc.cc.Write(req.Request) if err != nil { pc.close() return } ch := make(chan responseAndError, 1) - pc.reqch <- requestAndChan{req, ch, requestedGzip} + pc.reqch <- requestAndChan{req.Request, ch, requestedGzip} re := <-ch pc.lk.Lock() pc.numExpectedResponses-- @@ -634,7 +653,7 @@ func (pc *persistConn) close() { pc.broken = true pc.cc.Close() pc.conn.Close() - pc.mutateRequestFunc = nil + pc.mutateHeaderFunc = nil } var portMap = map[string]string{ diff --git a/libgo/go/http/transport_test.go b/libgo/go/http/transport_test.go index eafde7f..f3162b9 100644 --- a/libgo/go/http/transport_test.go +++ b/libgo/go/http/transport_test.go @@ -78,7 +78,7 @@ func TestTransportConnectionCloseOnResponse(t *testing.T) { fetch := func(n int) string { req := new(Request) var err os.Error - req.URL, err = url.Parse(ts.URL + fmt.Sprintf("?close=%v", connectionClose)) + req.URL, err = url.Parse(ts.URL + fmt.Sprintf("/?close=%v", connectionClose)) if err != nil { t.Fatalf("URL parse error: %v", err) } @@ -362,32 +362,6 @@ func TestTransportHeadChunkedResponse(t *testing.T) { } } -func TestTransportNilURL(t *testing.T) { - ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { - fmt.Fprintf(w, "Hi") - })) - defer ts.Close() - - req := new(Request) - req.URL = nil // what we're actually testing - req.Method = "GET" - req.RawURL = ts.URL - req.Proto = "HTTP/1.1" - req.ProtoMajor = 1 - req.ProtoMinor = 1 - req.Header = make(Header) - - tr := &Transport{} - res, err := tr.RoundTrip(req) - if err != nil { - t.Fatalf("unexpected RoundTrip error: %v", err) - } - body, err := ioutil.ReadAll(res.Body) - if g, e := string(body), "Hi"; g != e { - t.Fatalf("Expected response body of %q; got %q", e, g) - } -} - var roundTripTests = []struct { accept string expectAccept string @@ -398,7 +372,8 @@ var roundTripTests = []struct { // Requests with other accept-encoding should pass through unmodified {"foo", "foo", false}, // Requests with accept-encoding == gzip should be passed through - {"gzip", "gzip", true}} + {"gzip", "gzip", true}, +} // Test that the modification made to the Request by the RoundTripper is cleaned up func TestRoundTripGzip(t *testing.T) { @@ -406,7 +381,8 @@ func TestRoundTripGzip(t *testing.T) { ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { accept := req.Header.Get("Accept-Encoding") if expect := req.FormValue("expect_accept"); accept != expect { - t.Errorf("Accept-Encoding = %q, want %q", accept, expect) + t.Errorf("in handler, test %v: Accept-Encoding = %q, want %q", + req.FormValue("testnum"), accept, expect) } if accept == "gzip" { rw.Header().Set("Content-Encoding", "gzip") @@ -422,8 +398,10 @@ func TestRoundTripGzip(t *testing.T) { for i, test := range roundTripTests { // Test basic request (no accept-encoding) - req, _ := NewRequest("GET", ts.URL+"?expect_accept="+test.expectAccept, nil) - req.Header.Set("Accept-Encoding", test.accept) + req, _ := NewRequest("GET", fmt.Sprintf("%s/?testnum=%d&expect_accept=%s", ts.URL, i, test.expectAccept), nil) + if test.accept != "" { + req.Header.Set("Accept-Encoding", test.accept) + } res, err := DefaultTransport.RoundTrip(req) var body []byte if test.compressed { @@ -435,16 +413,16 @@ func TestRoundTripGzip(t *testing.T) { } if err != nil { t.Errorf("%d. Error: %q", i, err) - } else { - if g, e := string(body), responseBody; g != e { - t.Errorf("%d. body = %q; want %q", i, g, e) - } - if g, e := req.Header.Get("Accept-Encoding"), test.accept; g != e { - t.Errorf("%d. Accept-Encoding = %q; want %q", i, g, e) - } - if g, e := res.Header.Get("Content-Encoding"), test.accept; g != e { - t.Errorf("%d. Content-Encoding = %q; want %q", i, g, e) - } + continue + } + if g, e := string(body), responseBody; g != e { + t.Errorf("%d. body = %q; want %q", i, g, e) + } + if g, e := req.Header.Get("Accept-Encoding"), test.accept; g != e { + t.Errorf("%d. Accept-Encoding = %q; want %q (it was mutated, in violation of RoundTrip contract)", i, g, e) + } + if g, e := res.Header.Get("Content-Encoding"), test.accept; g != e { + t.Errorf("%d. Content-Encoding = %q; want %q", i, g, e) } } @@ -474,7 +452,7 @@ func TestTransportGzip(t *testing.T) { gz, _ := gzip.NewWriter(w) gz.Write([]byte(testString)) if req.FormValue("body") == "large" { - io.Copyn(gz, rand.Reader, nRandBytes) + io.CopyN(gz, rand.Reader, nRandBytes) } gz.Close() })) @@ -484,7 +462,7 @@ func TestTransportGzip(t *testing.T) { c := &Client{Transport: &Transport{}} // First fetch something large, but only read some of it. - res, err := c.Get(ts.URL + "?body=large&chunked=" + chunked) + res, err := c.Get(ts.URL + "/?body=large&chunked=" + chunked) if err != nil { t.Fatalf("large get: %v", err) } @@ -504,7 +482,7 @@ func TestTransportGzip(t *testing.T) { } // Then something small. - res, err = c.Get(ts.URL + "?chunked=" + chunked) + res, err = c.Get(ts.URL + "/?chunked=" + chunked) if err != nil { t.Fatal(err) } diff --git a/libgo/go/http/transport_windows.go b/libgo/go/http/transport_windows.go new file mode 100644 index 0000000..1ae7d83 --- /dev/null +++ b/libgo/go/http/transport_windows.go @@ -0,0 +1,21 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "os" + "net" +) + +func init() { + remoteSideClosedFunc = func(err os.Error) (out bool) { + op, ok := err.(*net.OpError) + if ok && op.Op == "WSARecv" && op.Net == "tcp" && op.Error == os.Errno(10058) { + // TODO(bradfitz): find the symbol for 10058 + return true + } + return false + } +} diff --git a/libgo/go/image/bmp/reader.go b/libgo/go/image/bmp/reader.go index 357da1d..134de5b 100644 --- a/libgo/go/image/bmp/reader.go +++ b/libgo/go/image/bmp/reader.go @@ -8,6 +8,7 @@ package bmp import ( + "image/color" "image" "io" "os" @@ -28,7 +29,7 @@ func readUint32(b []byte) uint32 { // decodePaletted reads an 8 bit-per-pixel BMP image from r. func decodePaletted(r io.Reader, c image.Config) (image.Image, os.Error) { var tmp [4]byte - paletted := image.NewPaletted(c.Width, c.Height, c.ColorModel.(image.PalettedColorModel)) + paletted := image.NewPaletted(image.Rect(0, 0, c.Width, c.Height), c.ColorModel.(color.Palette)) // BMP images are stored bottom-up rather than top-down. for y := c.Height - 1; y >= 0; y-- { p := paletted.Pix[y*paletted.Stride : y*paletted.Stride+c.Width] @@ -49,7 +50,7 @@ func decodePaletted(r io.Reader, c image.Config) (image.Image, os.Error) { // decodeRGBA reads a 24 bit-per-pixel BMP image from r. func decodeRGBA(r io.Reader, c image.Config) (image.Image, os.Error) { - rgba := image.NewRGBA(c.Width, c.Height) + rgba := image.NewRGBA(image.Rect(0, 0, c.Width, c.Height)) // There are 3 bytes per pixel, and each row is 4-byte aligned. b := make([]byte, (3*c.Width+3)&^3) // BMP images are stored bottom-up rather than top-down. @@ -77,7 +78,7 @@ func Decode(r io.Reader) (image.Image, os.Error) { if err != nil { return nil, err } - if c.ColorModel == image.RGBAColorModel { + if c.ColorModel == color.RGBAModel { return decodeRGBA(r, c) } return decodePaletted(r, c) @@ -128,11 +129,11 @@ func DecodeConfig(r io.Reader) (config image.Config, err os.Error) { if err != nil { return } - pcm := make(image.PalettedColorModel, 256) + pcm := make(color.Palette, 256) for i := range pcm { // BMP images are stored in BGR order rather than RGB order. // Every 4th byte is padding. - pcm[i] = image.RGBAColor{b[4*i+2], b[4*i+1], b[4*i+0], 0xFF} + pcm[i] = color.RGBA{b[4*i+2], b[4*i+1], b[4*i+0], 0xFF} } return image.Config{pcm, width, height}, nil case 24: @@ -140,7 +141,7 @@ func DecodeConfig(r io.Reader) (config image.Config, err os.Error) { err = ErrUnsupported return } - return image.Config{image.RGBAColorModel, width, height}, nil + return image.Config{color.RGBAModel, width, height}, nil } err = ErrUnsupported return diff --git a/libgo/go/image/color.go b/libgo/go/image/color.go deleted file mode 100644 index 501a882..0000000 --- a/libgo/go/image/color.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package image - -// Color can convert itself to alpha-premultiplied RGBA, with a possible loss -// of precision. Each value ranges within [0, 0xFFFF], but is represented by a -// uint32 so that multiplying by a blend factor up to 0xFFFF will not overflow. -type Color interface { - RGBA() (r, g, b, a uint32) -} - -// RGBAColor represents a traditional 32-bit alpha-premultiplied color, -// having 8 bits for each of red, green, blue and alpha. -type RGBAColor struct { - R, G, B, A uint8 -} - -func (c RGBAColor) RGBA() (r, g, b, a uint32) { - r = uint32(c.R) - r |= r << 8 - g = uint32(c.G) - g |= g << 8 - b = uint32(c.B) - b |= b << 8 - a = uint32(c.A) - a |= a << 8 - return -} - -// RGBA64Color represents a 64-bit alpha-premultiplied color, -// having 16 bits for each of red, green, blue and alpha. -type RGBA64Color struct { - R, G, B, A uint16 -} - -func (c RGBA64Color) RGBA() (r, g, b, a uint32) { - return uint32(c.R), uint32(c.G), uint32(c.B), uint32(c.A) -} - -// NRGBAColor represents a non-alpha-premultiplied 32-bit color. -type NRGBAColor struct { - R, G, B, A uint8 -} - -func (c NRGBAColor) RGBA() (r, g, b, a uint32) { - r = uint32(c.R) - r |= r << 8 - r *= uint32(c.A) - r /= 0xff - g = uint32(c.G) - g |= g << 8 - g *= uint32(c.A) - g /= 0xff - b = uint32(c.B) - b |= b << 8 - b *= uint32(c.A) - b /= 0xff - a = uint32(c.A) - a |= a << 8 - return -} - -// NRGBA64Color represents a non-alpha-premultiplied 64-bit color, -// having 16 bits for each of red, green, blue and alpha. -type NRGBA64Color struct { - R, G, B, A uint16 -} - -func (c NRGBA64Color) RGBA() (r, g, b, a uint32) { - r = uint32(c.R) - r *= uint32(c.A) - r /= 0xffff - g = uint32(c.G) - g *= uint32(c.A) - g /= 0xffff - b = uint32(c.B) - b *= uint32(c.A) - b /= 0xffff - a = uint32(c.A) - return -} - -// AlphaColor represents an 8-bit alpha. -type AlphaColor struct { - A uint8 -} - -func (c AlphaColor) RGBA() (r, g, b, a uint32) { - a = uint32(c.A) - a |= a << 8 - return a, a, a, a -} - -// Alpha16Color represents a 16-bit alpha. -type Alpha16Color struct { - A uint16 -} - -func (c Alpha16Color) RGBA() (r, g, b, a uint32) { - a = uint32(c.A) - return a, a, a, a -} - -// GrayColor represents an 8-bit grayscale color. -type GrayColor struct { - Y uint8 -} - -func (c GrayColor) RGBA() (r, g, b, a uint32) { - y := uint32(c.Y) - y |= y << 8 - return y, y, y, 0xffff -} - -// Gray16Color represents a 16-bit grayscale color. -type Gray16Color struct { - Y uint16 -} - -func (c Gray16Color) RGBA() (r, g, b, a uint32) { - y := uint32(c.Y) - return y, y, y, 0xffff -} - -// ColorModel can convert foreign Colors, with a possible loss of precision, -// to a Color from its own color model. -type ColorModel interface { - Convert(c Color) Color -} - -// The ColorModelFunc type is an adapter to allow the use of an ordinary -// color conversion function as a ColorModel. If f is such a function, -// ColorModelFunc(f) is a ColorModel object that invokes f to implement -// the conversion. -type ColorModelFunc func(Color) Color - -func (f ColorModelFunc) Convert(c Color) Color { - return f(c) -} - -func toRGBAColor(c Color) Color { - if _, ok := c.(RGBAColor); ok { - return c - } - r, g, b, a := c.RGBA() - return RGBAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)} -} - -func toRGBA64Color(c Color) Color { - if _, ok := c.(RGBA64Color); ok { - return c - } - r, g, b, a := c.RGBA() - return RGBA64Color{uint16(r), uint16(g), uint16(b), uint16(a)} -} - -func toNRGBAColor(c Color) Color { - if _, ok := c.(NRGBAColor); ok { - return c - } - r, g, b, a := c.RGBA() - if a == 0xffff { - return NRGBAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), 0xff} - } - if a == 0 { - return NRGBAColor{0, 0, 0, 0} - } - // Since Color.RGBA returns a alpha-premultiplied color, we should have r <= a && g <= a && b <= a. - r = (r * 0xffff) / a - g = (g * 0xffff) / a - b = (b * 0xffff) / a - return NRGBAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)} -} - -func toNRGBA64Color(c Color) Color { - if _, ok := c.(NRGBA64Color); ok { - return c - } - r, g, b, a := c.RGBA() - if a == 0xffff { - return NRGBA64Color{uint16(r), uint16(g), uint16(b), 0xffff} - } - if a == 0 { - return NRGBA64Color{0, 0, 0, 0} - } - // Since Color.RGBA returns a alpha-premultiplied color, we should have r <= a && g <= a && b <= a. - r = (r * 0xffff) / a - g = (g * 0xffff) / a - b = (b * 0xffff) / a - return NRGBA64Color{uint16(r), uint16(g), uint16(b), uint16(a)} -} - -func toAlphaColor(c Color) Color { - if _, ok := c.(AlphaColor); ok { - return c - } - _, _, _, a := c.RGBA() - return AlphaColor{uint8(a >> 8)} -} - -func toAlpha16Color(c Color) Color { - if _, ok := c.(Alpha16Color); ok { - return c - } - _, _, _, a := c.RGBA() - return Alpha16Color{uint16(a)} -} - -func toGrayColor(c Color) Color { - if _, ok := c.(GrayColor); ok { - return c - } - r, g, b, _ := c.RGBA() - y := (299*r + 587*g + 114*b + 500) / 1000 - return GrayColor{uint8(y >> 8)} -} - -func toGray16Color(c Color) Color { - if _, ok := c.(Gray16Color); ok { - return c - } - r, g, b, _ := c.RGBA() - y := (299*r + 587*g + 114*b + 500) / 1000 - return Gray16Color{uint16(y)} -} - -// The ColorModel associated with RGBAColor. -var RGBAColorModel ColorModel = ColorModelFunc(toRGBAColor) - -// The ColorModel associated with RGBA64Color. -var RGBA64ColorModel ColorModel = ColorModelFunc(toRGBA64Color) - -// The ColorModel associated with NRGBAColor. -var NRGBAColorModel ColorModel = ColorModelFunc(toNRGBAColor) - -// The ColorModel associated with NRGBA64Color. -var NRGBA64ColorModel ColorModel = ColorModelFunc(toNRGBA64Color) - -// The ColorModel associated with AlphaColor. -var AlphaColorModel ColorModel = ColorModelFunc(toAlphaColor) - -// The ColorModel associated with Alpha16Color. -var Alpha16ColorModel ColorModel = ColorModelFunc(toAlpha16Color) - -// The ColorModel associated with GrayColor. -var GrayColorModel ColorModel = ColorModelFunc(toGrayColor) - -// The ColorModel associated with Gray16Color. -var Gray16ColorModel ColorModel = ColorModelFunc(toGray16Color) diff --git a/libgo/go/image/color/color.go b/libgo/go/image/color/color.go new file mode 100644 index 0000000..4a0fae5 --- /dev/null +++ b/libgo/go/image/color/color.go @@ -0,0 +1,293 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package color implements a basic color library. +package color + +// Color can convert itself to alpha-premultiplied 16-bits per channel RGBA. +// The conversion may be lossy. +type Color interface { + // RGBA returns the alpha-premultiplied red, green, blue and alpha values + // for the color. Each value ranges within [0, 0xFFFF], but is represented + // by a uint32 so that multiplying by a blend factor up to 0xFFFF will not + // overflow. + RGBA() (r, g, b, a uint32) +} + +// RGBA represents a traditional 32-bit alpha-premultiplied color, +// having 8 bits for each of red, green, blue and alpha. +type RGBA struct { + R, G, B, A uint8 +} + +func (c RGBA) RGBA() (r, g, b, a uint32) { + r = uint32(c.R) + r |= r << 8 + g = uint32(c.G) + g |= g << 8 + b = uint32(c.B) + b |= b << 8 + a = uint32(c.A) + a |= a << 8 + return +} + +// RGBA64 represents a 64-bit alpha-premultiplied color, +// having 16 bits for each of red, green, blue and alpha. +type RGBA64 struct { + R, G, B, A uint16 +} + +func (c RGBA64) RGBA() (r, g, b, a uint32) { + return uint32(c.R), uint32(c.G), uint32(c.B), uint32(c.A) +} + +// NRGBA represents a non-alpha-premultiplied 32-bit color. +type NRGBA struct { + R, G, B, A uint8 +} + +func (c NRGBA) RGBA() (r, g, b, a uint32) { + r = uint32(c.R) + r |= r << 8 + r *= uint32(c.A) + r /= 0xff + g = uint32(c.G) + g |= g << 8 + g *= uint32(c.A) + g /= 0xff + b = uint32(c.B) + b |= b << 8 + b *= uint32(c.A) + b /= 0xff + a = uint32(c.A) + a |= a << 8 + return +} + +// NRGBA64 represents a non-alpha-premultiplied 64-bit color, +// having 16 bits for each of red, green, blue and alpha. +type NRGBA64 struct { + R, G, B, A uint16 +} + +func (c NRGBA64) RGBA() (r, g, b, a uint32) { + r = uint32(c.R) + r *= uint32(c.A) + r /= 0xffff + g = uint32(c.G) + g *= uint32(c.A) + g /= 0xffff + b = uint32(c.B) + b *= uint32(c.A) + b /= 0xffff + a = uint32(c.A) + return +} + +// Alpha represents an 8-bit alpha color. +type Alpha struct { + A uint8 +} + +func (c Alpha) RGBA() (r, g, b, a uint32) { + a = uint32(c.A) + a |= a << 8 + return a, a, a, a +} + +// Alpha16 represents a 16-bit alpha color. +type Alpha16 struct { + A uint16 +} + +func (c Alpha16) RGBA() (r, g, b, a uint32) { + a = uint32(c.A) + return a, a, a, a +} + +// Gray represents an 8-bit grayscale color. +type Gray struct { + Y uint8 +} + +func (c Gray) RGBA() (r, g, b, a uint32) { + y := uint32(c.Y) + y |= y << 8 + return y, y, y, 0xffff +} + +// Gray16 represents a 16-bit grayscale color. +type Gray16 struct { + Y uint16 +} + +func (c Gray16) RGBA() (r, g, b, a uint32) { + y := uint32(c.Y) + return y, y, y, 0xffff +} + +// Model can convert any Color to one from its own color model. The conversion +// may be lossy. +type Model interface { + Convert(c Color) Color +} + +// ModelFunc is an adapter type to allow the use of a color conversion +// function as a Model. If f is such a function, ModelFunc(f) is a Model that +// invokes f to implement the conversion. +type ModelFunc func(Color) Color + +func (f ModelFunc) Convert(c Color) Color { + return f(c) +} + +// RGBAModel is the Model for RGBA colors. +var RGBAModel Model = ModelFunc(func(c Color) Color { + if _, ok := c.(RGBA); ok { + return c + } + r, g, b, a := c.RGBA() + return RGBA{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)} +}) + +// RGBAModel is the Model for RGBA64 colors. +var RGBA64Model Model = ModelFunc(func(c Color) Color { + if _, ok := c.(RGBA64); ok { + return c + } + r, g, b, a := c.RGBA() + return RGBA64{uint16(r), uint16(g), uint16(b), uint16(a)} +}) + +// NRGBAModel is the Model for NRGBA colors. +var NRGBAModel Model = ModelFunc(func(c Color) Color { + if _, ok := c.(NRGBA); ok { + return c + } + r, g, b, a := c.RGBA() + if a == 0xffff { + return NRGBA{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), 0xff} + } + if a == 0 { + return NRGBA{0, 0, 0, 0} + } + // Since Color.RGBA returns a alpha-premultiplied color, we should have r <= a && g <= a && b <= a. + r = (r * 0xffff) / a + g = (g * 0xffff) / a + b = (b * 0xffff) / a + return NRGBA{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)} +}) + +// NRGBAModel is the Model for NRGBA64 colors. +var NRGBA64Model Model = ModelFunc(func(c Color) Color { + if _, ok := c.(NRGBA64); ok { + return c + } + r, g, b, a := c.RGBA() + if a == 0xffff { + return NRGBA64{uint16(r), uint16(g), uint16(b), 0xffff} + } + if a == 0 { + return NRGBA64{0, 0, 0, 0} + } + // Since Color.RGBA returns a alpha-premultiplied color, we should have r <= a && g <= a && b <= a. + r = (r * 0xffff) / a + g = (g * 0xffff) / a + b = (b * 0xffff) / a + return NRGBA64{uint16(r), uint16(g), uint16(b), uint16(a)} +}) + +// AlphaModel is the Model for Alpha colors. +var AlphaModel Model = ModelFunc(func(c Color) Color { + if _, ok := c.(Alpha); ok { + return c + } + _, _, _, a := c.RGBA() + return Alpha{uint8(a >> 8)} +}) + +// Alpha16Model is the Model for Alpha16 colors. +var Alpha16Model Model = ModelFunc(func(c Color) Color { + if _, ok := c.(Alpha16); ok { + return c + } + _, _, _, a := c.RGBA() + return Alpha16{uint16(a)} +}) + +// GrayModel is the Model for Gray colors. +var GrayModel Model = ModelFunc(func(c Color) Color { + if _, ok := c.(Gray); ok { + return c + } + r, g, b, _ := c.RGBA() + y := (299*r + 587*g + 114*b + 500) / 1000 + return Gray{uint8(y >> 8)} +}) + +// Gray16Model is the Model for Gray16 colors. +var Gray16Model Model = ModelFunc(func(c Color) Color { + if _, ok := c.(Gray16); ok { + return c + } + r, g, b, _ := c.RGBA() + y := (299*r + 587*g + 114*b + 500) / 1000 + return Gray16{uint16(y)} +}) + +// Palette is a palette of colors. +type Palette []Color + +func diff(a, b uint32) uint32 { + if a > b { + return a - b + } + return b - a +} + +// Convert returns the palette color closest to c in Euclidean R,G,B space. +func (p Palette) Convert(c Color) Color { + if len(p) == 0 { + return nil + } + return p[p.Index(c)] +} + +// Index returns the index of the palette color closest to c in Euclidean +// R,G,B space. +func (p Palette) Index(c Color) int { + cr, cg, cb, _ := c.RGBA() + // Shift by 1 bit to avoid potential uint32 overflow in sum-squared-difference. + cr >>= 1 + cg >>= 1 + cb >>= 1 + ret, bestSSD := 0, uint32(1<<32-1) + for i, v := range p { + vr, vg, vb, _ := v.RGBA() + vr >>= 1 + vg >>= 1 + vb >>= 1 + dr, dg, db := diff(cr, vr), diff(cg, vg), diff(cb, vb) + ssd := (dr * dr) + (dg * dg) + (db * db) + if ssd < bestSSD { + if ssd == 0 { + return i + } + ret, bestSSD = i, ssd + } + } + return ret +} + +var ( + // Black is an opaque black Color. + Black = Gray16{0} + // White is an opaque white Color. + White = Gray16{0xffff} + // Transparent is a fully transparent Color. + Transparent = Alpha16{0} + // Opaque is a fully opaque Color. + Opaque = Alpha16{0xffff} +) diff --git a/libgo/go/image/decode_test.go b/libgo/go/image/decode_test.go index 540d5ed..b348c1d 100644 --- a/libgo/go/image/decode_test.go +++ b/libgo/go/image/decode_test.go @@ -7,6 +7,7 @@ package image_test import ( "bufio" "image" + "image/color" "os" "testing" @@ -66,7 +67,7 @@ func delta(u0, u1 uint32) int { return d } -func withinTolerance(c0, c1 image.Color, tolerance int) bool { +func withinTolerance(c0, c1 color.Color, tolerance int) bool { r0, g0, b0, a0 := c0.RGBA() r1, g1, b1, a1 := c1.RGBA() r := delta(r0, r1) diff --git a/libgo/go/image/draw/bench_test.go b/libgo/go/image/draw/bench_test.go index a99b408..2be9118 100644 --- a/libgo/go/image/draw/bench_test.go +++ b/libgo/go/image/draw/bench_test.go @@ -6,6 +6,7 @@ package draw import ( "image" + "image/color" "image/ycbcr" "testing" ) @@ -18,16 +19,16 @@ const ( // bench benchmarks drawing src and mask images onto a dst image with the // given op and the color models to create those images from. // The created images' pixels are initialized to non-zero values. -func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { +func bench(b *testing.B, dcm, scm, mcm color.Model, op Op) { b.StopTimer() var dst Image switch dcm { - case image.RGBAColorModel: - dst1 := image.NewRGBA(dstw, dsth) + case color.RGBAModel: + dst1 := image.NewRGBA(image.Rect(0, 0, dstw, dsth)) for y := 0; y < dsth; y++ { for x := 0; x < dstw; x++ { - dst1.SetRGBA(x, y, image.RGBAColor{ + dst1.SetRGBA(x, y, color.RGBA{ uint8(5 * x % 0x100), uint8(7 * y % 0x100), uint8((7*x + 5*y) % 0x100), @@ -36,11 +37,11 @@ func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { } } dst = dst1 - case image.RGBA64ColorModel: - dst1 := image.NewRGBA64(dstw, dsth) + case color.RGBA64Model: + dst1 := image.NewRGBA64(image.Rect(0, 0, dstw, dsth)) for y := 0; y < dsth; y++ { for x := 0; x < dstw; x++ { - dst1.SetRGBA64(x, y, image.RGBA64Color{ + dst1.SetRGBA64(x, y, color.RGBA64{ uint16(53 * x % 0x10000), uint16(59 * y % 0x10000), uint16((59*x + 53*y) % 0x10000), @@ -56,12 +57,12 @@ func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { var src image.Image switch scm { case nil: - src = &image.ColorImage{image.RGBAColor{0x11, 0x22, 0x33, 0xff}} - case image.RGBAColorModel: - src1 := image.NewRGBA(srcw, srch) + src = &image.Uniform{color.RGBA{0x11, 0x22, 0x33, 0xff}} + case color.RGBAModel: + src1 := image.NewRGBA(image.Rect(0, 0, srcw, srch)) for y := 0; y < srch; y++ { for x := 0; x < srcw; x++ { - src1.SetRGBA(x, y, image.RGBAColor{ + src1.SetRGBA(x, y, color.RGBA{ uint8(13 * x % 0x80), uint8(11 * y % 0x80), uint8((11*x + 13*y) % 0x80), @@ -70,11 +71,11 @@ func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { } } src = src1 - case image.RGBA64ColorModel: - src1 := image.NewRGBA64(srcw, srch) + case color.RGBA64Model: + src1 := image.NewRGBA64(image.Rect(0, 0, srcw, srch)) for y := 0; y < srch; y++ { for x := 0; x < srcw; x++ { - src1.SetRGBA64(x, y, image.RGBA64Color{ + src1.SetRGBA64(x, y, color.RGBA64{ uint16(103 * x % 0x8000), uint16(101 * y % 0x8000), uint16((101*x + 103*y) % 0x8000), @@ -83,11 +84,11 @@ func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { } } src = src1 - case image.NRGBAColorModel: - src1 := image.NewNRGBA(srcw, srch) + case color.NRGBAModel: + src1 := image.NewNRGBA(image.Rect(0, 0, srcw, srch)) for y := 0; y < srch; y++ { for x := 0; x < srcw; x++ { - src1.SetNRGBA(x, y, image.NRGBAColor{ + src1.SetNRGBA(x, y, color.NRGBA{ uint8(13 * x % 0x100), uint8(11 * y % 0x100), uint8((11*x + 13*y) % 0x100), @@ -122,15 +123,15 @@ func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { switch mcm { case nil: // No-op. - case image.AlphaColorModel: - mask1 := image.NewAlpha(srcw, srch) + case color.AlphaModel: + mask1 := image.NewAlpha(image.Rect(0, 0, srcw, srch)) for y := 0; y < srch; y++ { for x := 0; x < srcw; x++ { a := uint8((23*x + 29*y) % 0x100) // Glyph masks are typically mostly zero, // so we only set a quarter of mask1's pixels. if a >= 0xc0 { - mask1.SetAlpha(x, y, image.AlphaColor{a}) + mask1.SetAlpha(x, y, color.Alpha{a}) } } } @@ -152,55 +153,55 @@ func bench(b *testing.B, dcm, scm, mcm image.ColorModel, op Op) { // The BenchmarkFoo functions exercise a drawFoo fast-path function in draw.go. func BenchmarkFillOver(b *testing.B) { - bench(b, image.RGBAColorModel, nil, nil, Over) + bench(b, color.RGBAModel, nil, nil, Over) } func BenchmarkFillSrc(b *testing.B) { - bench(b, image.RGBAColorModel, nil, nil, Src) + bench(b, color.RGBAModel, nil, nil, Src) } func BenchmarkCopyOver(b *testing.B) { - bench(b, image.RGBAColorModel, image.RGBAColorModel, nil, Over) + bench(b, color.RGBAModel, color.RGBAModel, nil, Over) } func BenchmarkCopySrc(b *testing.B) { - bench(b, image.RGBAColorModel, image.RGBAColorModel, nil, Src) + bench(b, color.RGBAModel, color.RGBAModel, nil, Src) } func BenchmarkNRGBAOver(b *testing.B) { - bench(b, image.RGBAColorModel, image.NRGBAColorModel, nil, Over) + bench(b, color.RGBAModel, color.NRGBAModel, nil, Over) } func BenchmarkNRGBASrc(b *testing.B) { - bench(b, image.RGBAColorModel, image.NRGBAColorModel, nil, Src) + bench(b, color.RGBAModel, color.NRGBAModel, nil, Src) } func BenchmarkYCbCr(b *testing.B) { - bench(b, image.RGBAColorModel, ycbcr.YCbCrColorModel, nil, Over) + bench(b, color.RGBAModel, ycbcr.YCbCrColorModel, nil, Over) } func BenchmarkGlyphOver(b *testing.B) { - bench(b, image.RGBAColorModel, nil, image.AlphaColorModel, Over) + bench(b, color.RGBAModel, nil, color.AlphaModel, Over) } func BenchmarkRGBA(b *testing.B) { - bench(b, image.RGBAColorModel, image.RGBA64ColorModel, nil, Src) + bench(b, color.RGBAModel, color.RGBA64Model, nil, Src) } // The BenchmarkGenericFoo functions exercise the generic, slow-path code. func BenchmarkGenericOver(b *testing.B) { - bench(b, image.RGBA64ColorModel, image.RGBA64ColorModel, nil, Over) + bench(b, color.RGBA64Model, color.RGBA64Model, nil, Over) } func BenchmarkGenericMaskOver(b *testing.B) { - bench(b, image.RGBA64ColorModel, image.RGBA64ColorModel, image.AlphaColorModel, Over) + bench(b, color.RGBA64Model, color.RGBA64Model, color.AlphaModel, Over) } func BenchmarkGenericSrc(b *testing.B) { - bench(b, image.RGBA64ColorModel, image.RGBA64ColorModel, nil, Src) + bench(b, color.RGBA64Model, color.RGBA64Model, nil, Src) } func BenchmarkGenericMaskSrc(b *testing.B) { - bench(b, image.RGBA64ColorModel, image.RGBA64ColorModel, image.AlphaColorModel, Src) + bench(b, color.RGBA64Model, color.RGBA64Model, color.AlphaModel, Src) } diff --git a/libgo/go/image/draw/clip_test.go b/libgo/go/image/draw/clip_test.go index db40d82..65381f7 100644 --- a/libgo/go/image/draw/clip_test.go +++ b/libgo/go/image/draw/clip_test.go @@ -143,9 +143,9 @@ var clipTests = []clipTest{ } func TestClip(t *testing.T) { - dst0 := image.NewRGBA(100, 100) - src0 := image.NewRGBA(100, 100) - mask0 := image.NewRGBA(100, 100) + dst0 := image.NewRGBA(image.Rect(0, 0, 100, 100)) + src0 := image.NewRGBA(image.Rect(0, 0, 100, 100)) + mask0 := image.NewRGBA(image.Rect(0, 0, 100, 100)) for _, c := range clipTests { dst := dst0.SubImage(c.dr).(*image.RGBA) src := src0.SubImage(c.sr).(*image.RGBA) diff --git a/libgo/go/image/draw/draw.go b/libgo/go/image/draw/draw.go index 5171e03..af02639 100644 --- a/libgo/go/image/draw/draw.go +++ b/libgo/go/image/draw/draw.go @@ -10,6 +10,7 @@ package draw import ( "image" + "image/color" "image/ycbcr" ) @@ -26,12 +27,10 @@ const ( Src ) -var zeroColor image.Color = image.AlphaColor{0} - // A draw.Image is an image.Image with a Set method to change a single pixel. type Image interface { image.Image - Set(x, y int, c image.Color) + Set(x, y int, c color.Color) } // Draw calls DrawMask with a nil mask. @@ -73,7 +72,7 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas if op == Over { if mask == nil { switch src0 := src.(type) { - case *image.ColorImage: + case *image.Uniform: drawFillOver(dst0, r, src0) return case *image.RGBA: @@ -88,7 +87,7 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas } } else if mask0, ok := mask.(*image.Alpha); ok { switch src0 := src.(type) { - case *image.ColorImage: + case *image.Uniform: drawGlyphOver(dst0, r, src0, mask0, mp) return } @@ -96,7 +95,7 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas } else { if mask == nil { switch src0 := src.(type) { - case *image.ColorImage: + case *image.Uniform: drawFillSrc(dst0, r, src0) return case *image.RGBA: @@ -125,7 +124,7 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas } } - var out *image.RGBA64Color + var out *color.RGBA64 sy := sp.Y + y0 - r.Min.Y my := mp.Y + y0 - r.Min.Y for y := y0; y != y1; y, sy, my = y+dy, sy+dy, my+dy { @@ -141,14 +140,14 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas if op == Over { // No-op. } else { - dst.Set(x, y, zeroColor) + dst.Set(x, y, color.Transparent) } case ma == m && op == Src: dst.Set(x, y, src.At(sx, sy)) default: sr, sg, sb, sa := src.At(sx, sy).RGBA() if out == nil { - out = new(image.RGBA64Color) + out = new(color.RGBA64) } if op == Over { dr, dg, db, da := dst.At(x, y).RGBA() @@ -169,7 +168,7 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas } } -func drawFillOver(dst *image.RGBA, r image.Rectangle, src *image.ColorImage) { +func drawFillOver(dst *image.RGBA, r image.Rectangle, src *image.Uniform) { sr, sg, sb, sa := src.RGBA() // The 0x101 is here for the same reason as in drawRGBA. a := (m - sa) * 0x101 @@ -192,7 +191,7 @@ func drawFillOver(dst *image.RGBA, r image.Rectangle, src *image.ColorImage) { } } -func drawFillSrc(dst *image.RGBA, r image.Rectangle, src *image.ColorImage) { +func drawFillSrc(dst *image.RGBA, r image.Rectangle, src *image.Uniform) { sr, sg, sb, sa := src.RGBA() // The built-in copy function is faster than a straightforward for loop to fill the destination with // the color, but copy requires a slice source. We therefore use a for loop to fill the first row, and @@ -406,7 +405,7 @@ func drawYCbCr(dst *image.RGBA, r image.Rectangle, src *ycbcr.YCbCr, sp image.Po } } -func drawGlyphOver(dst *image.RGBA, r image.Rectangle, src *image.ColorImage, mask *image.Alpha, mp image.Point) { +func drawGlyphOver(dst *image.RGBA, r image.Rectangle, src *image.Uniform, mask *image.Alpha, mp image.Point) { i0 := (r.Min.Y-dst.Rect.Min.Y)*dst.Stride + (r.Min.X-dst.Rect.Min.X)*4 i1 := i0 + r.Dx()*4 mi0 := (mp.Y-mask.Rect.Min.Y)*mask.Stride + mp.X - mask.Rect.Min.X diff --git a/libgo/go/image/draw/draw_test.go b/libgo/go/image/draw/draw_test.go index 55435cc..663ab67 100644 --- a/libgo/go/image/draw/draw_test.go +++ b/libgo/go/image/draw/draw_test.go @@ -6,49 +6,50 @@ package draw import ( "image" + "image/color" "image/ycbcr" "testing" ) -func eq(c0, c1 image.Color) bool { +func eq(c0, c1 color.Color) bool { r0, g0, b0, a0 := c0.RGBA() r1, g1, b1, a1 := c1.RGBA() return r0 == r1 && g0 == g1 && b0 == b1 && a0 == a1 } func fillBlue(alpha int) image.Image { - return image.NewColorImage(image.RGBAColor{0, 0, uint8(alpha), uint8(alpha)}) + return image.NewUniform(color.RGBA{0, 0, uint8(alpha), uint8(alpha)}) } func fillAlpha(alpha int) image.Image { - return image.NewColorImage(image.AlphaColor{uint8(alpha)}) + return image.NewUniform(color.Alpha{uint8(alpha)}) } func vgradGreen(alpha int) image.Image { - m := image.NewRGBA(16, 16) + m := image.NewRGBA(image.Rect(0, 0, 16, 16)) for y := 0; y < 16; y++ { for x := 0; x < 16; x++ { - m.Set(x, y, image.RGBAColor{0, uint8(y * alpha / 15), 0, uint8(alpha)}) + m.Set(x, y, color.RGBA{0, uint8(y * alpha / 15), 0, uint8(alpha)}) } } return m } func vgradAlpha(alpha int) image.Image { - m := image.NewAlpha(16, 16) + m := image.NewAlpha(image.Rect(0, 0, 16, 16)) for y := 0; y < 16; y++ { for x := 0; x < 16; x++ { - m.Set(x, y, image.AlphaColor{uint8(y * alpha / 15)}) + m.Set(x, y, color.Alpha{uint8(y * alpha / 15)}) } } return m } func vgradGreenNRGBA(alpha int) image.Image { - m := image.NewNRGBA(16, 16) + m := image.NewNRGBA(image.Rect(0, 0, 16, 16)) for y := 0; y < 16; y++ { for x := 0; x < 16; x++ { - m.Set(x, y, image.RGBAColor{0, uint8(y * 0x11), 0, uint8(alpha)}) + m.Set(x, y, color.RGBA{0, uint8(y * 0x11), 0, uint8(alpha)}) } } return m @@ -73,20 +74,20 @@ func vgradCr() image.Image { } func hgradRed(alpha int) Image { - m := image.NewRGBA(16, 16) + m := image.NewRGBA(image.Rect(0, 0, 16, 16)) for y := 0; y < 16; y++ { for x := 0; x < 16; x++ { - m.Set(x, y, image.RGBAColor{uint8(x * alpha / 15), 0, 0, uint8(alpha)}) + m.Set(x, y, color.RGBA{uint8(x * alpha / 15), 0, 0, uint8(alpha)}) } } return m } func gradYellow(alpha int) Image { - m := image.NewRGBA(16, 16) + m := image.NewRGBA(image.Rect(0, 0, 16, 16)) for y := 0; y < 16; y++ { for x := 0; x < 16; x++ { - m.Set(x, y, image.RGBAColor{uint8(x * alpha / 15), uint8(y * alpha / 15), 0, uint8(alpha)}) + m.Set(x, y, color.RGBA{uint8(x * alpha / 15), uint8(y * alpha / 15), 0, uint8(alpha)}) } } return m @@ -97,61 +98,61 @@ type drawTest struct { src image.Image mask image.Image op Op - expected image.Color + expected color.Color } var drawTests = []drawTest{ // Uniform mask (0% opaque). - {"nop", vgradGreen(255), fillAlpha(0), Over, image.RGBAColor{136, 0, 0, 255}}, - {"clear", vgradGreen(255), fillAlpha(0), Src, image.RGBAColor{0, 0, 0, 0}}, + {"nop", vgradGreen(255), fillAlpha(0), Over, color.RGBA{136, 0, 0, 255}}, + {"clear", vgradGreen(255), fillAlpha(0), Src, color.RGBA{0, 0, 0, 0}}, // Uniform mask (100%, 75%, nil) and uniform source. // At (x, y) == (8, 8): // The destination pixel is {136, 0, 0, 255}. // The source pixel is {0, 0, 90, 90}. - {"fill", fillBlue(90), fillAlpha(255), Over, image.RGBAColor{88, 0, 90, 255}}, - {"fillSrc", fillBlue(90), fillAlpha(255), Src, image.RGBAColor{0, 0, 90, 90}}, - {"fillAlpha", fillBlue(90), fillAlpha(192), Over, image.RGBAColor{100, 0, 68, 255}}, - {"fillAlphaSrc", fillBlue(90), fillAlpha(192), Src, image.RGBAColor{0, 0, 68, 68}}, - {"fillNil", fillBlue(90), nil, Over, image.RGBAColor{88, 0, 90, 255}}, - {"fillNilSrc", fillBlue(90), nil, Src, image.RGBAColor{0, 0, 90, 90}}, + {"fill", fillBlue(90), fillAlpha(255), Over, color.RGBA{88, 0, 90, 255}}, + {"fillSrc", fillBlue(90), fillAlpha(255), Src, color.RGBA{0, 0, 90, 90}}, + {"fillAlpha", fillBlue(90), fillAlpha(192), Over, color.RGBA{100, 0, 68, 255}}, + {"fillAlphaSrc", fillBlue(90), fillAlpha(192), Src, color.RGBA{0, 0, 68, 68}}, + {"fillNil", fillBlue(90), nil, Over, color.RGBA{88, 0, 90, 255}}, + {"fillNilSrc", fillBlue(90), nil, Src, color.RGBA{0, 0, 90, 90}}, // Uniform mask (100%, 75%, nil) and variable source. // At (x, y) == (8, 8): // The destination pixel is {136, 0, 0, 255}. // The source pixel is {0, 48, 0, 90}. - {"copy", vgradGreen(90), fillAlpha(255), Over, image.RGBAColor{88, 48, 0, 255}}, - {"copySrc", vgradGreen(90), fillAlpha(255), Src, image.RGBAColor{0, 48, 0, 90}}, - {"copyAlpha", vgradGreen(90), fillAlpha(192), Over, image.RGBAColor{100, 36, 0, 255}}, - {"copyAlphaSrc", vgradGreen(90), fillAlpha(192), Src, image.RGBAColor{0, 36, 0, 68}}, - {"copyNil", vgradGreen(90), nil, Over, image.RGBAColor{88, 48, 0, 255}}, - {"copyNilSrc", vgradGreen(90), nil, Src, image.RGBAColor{0, 48, 0, 90}}, + {"copy", vgradGreen(90), fillAlpha(255), Over, color.RGBA{88, 48, 0, 255}}, + {"copySrc", vgradGreen(90), fillAlpha(255), Src, color.RGBA{0, 48, 0, 90}}, + {"copyAlpha", vgradGreen(90), fillAlpha(192), Over, color.RGBA{100, 36, 0, 255}}, + {"copyAlphaSrc", vgradGreen(90), fillAlpha(192), Src, color.RGBA{0, 36, 0, 68}}, + {"copyNil", vgradGreen(90), nil, Over, color.RGBA{88, 48, 0, 255}}, + {"copyNilSrc", vgradGreen(90), nil, Src, color.RGBA{0, 48, 0, 90}}, // Uniform mask (100%, 75%, nil) and variable NRGBA source. // At (x, y) == (8, 8): // The destination pixel is {136, 0, 0, 255}. // The source pixel is {0, 136, 0, 90} in NRGBA-space, which is {0, 48, 0, 90} in RGBA-space. // The result pixel is different than in the "copy*" test cases because of rounding errors. - {"nrgba", vgradGreenNRGBA(90), fillAlpha(255), Over, image.RGBAColor{88, 46, 0, 255}}, - {"nrgbaSrc", vgradGreenNRGBA(90), fillAlpha(255), Src, image.RGBAColor{0, 46, 0, 90}}, - {"nrgbaAlpha", vgradGreenNRGBA(90), fillAlpha(192), Over, image.RGBAColor{100, 34, 0, 255}}, - {"nrgbaAlphaSrc", vgradGreenNRGBA(90), fillAlpha(192), Src, image.RGBAColor{0, 34, 0, 68}}, - {"nrgbaNil", vgradGreenNRGBA(90), nil, Over, image.RGBAColor{88, 46, 0, 255}}, - {"nrgbaNilSrc", vgradGreenNRGBA(90), nil, Src, image.RGBAColor{0, 46, 0, 90}}, + {"nrgba", vgradGreenNRGBA(90), fillAlpha(255), Over, color.RGBA{88, 46, 0, 255}}, + {"nrgbaSrc", vgradGreenNRGBA(90), fillAlpha(255), Src, color.RGBA{0, 46, 0, 90}}, + {"nrgbaAlpha", vgradGreenNRGBA(90), fillAlpha(192), Over, color.RGBA{100, 34, 0, 255}}, + {"nrgbaAlphaSrc", vgradGreenNRGBA(90), fillAlpha(192), Src, color.RGBA{0, 34, 0, 68}}, + {"nrgbaNil", vgradGreenNRGBA(90), nil, Over, color.RGBA{88, 46, 0, 255}}, + {"nrgbaNilSrc", vgradGreenNRGBA(90), nil, Src, color.RGBA{0, 46, 0, 90}}, // Uniform mask (100%, 75%, nil) and variable YCbCr source. // At (x, y) == (8, 8): // The destination pixel is {136, 0, 0, 255}. // The source pixel is {0, 0, 136} in YCbCr-space, which is {11, 38, 0, 255} in RGB-space. - {"ycbcr", vgradCr(), fillAlpha(255), Over, image.RGBAColor{11, 38, 0, 255}}, - {"ycbcrSrc", vgradCr(), fillAlpha(255), Src, image.RGBAColor{11, 38, 0, 255}}, - {"ycbcrAlpha", vgradCr(), fillAlpha(192), Over, image.RGBAColor{42, 28, 0, 255}}, - {"ycbcrAlphaSrc", vgradCr(), fillAlpha(192), Src, image.RGBAColor{8, 28, 0, 192}}, - {"ycbcrNil", vgradCr(), nil, Over, image.RGBAColor{11, 38, 0, 255}}, - {"ycbcrNilSrc", vgradCr(), nil, Src, image.RGBAColor{11, 38, 0, 255}}, + {"ycbcr", vgradCr(), fillAlpha(255), Over, color.RGBA{11, 38, 0, 255}}, + {"ycbcrSrc", vgradCr(), fillAlpha(255), Src, color.RGBA{11, 38, 0, 255}}, + {"ycbcrAlpha", vgradCr(), fillAlpha(192), Over, color.RGBA{42, 28, 0, 255}}, + {"ycbcrAlphaSrc", vgradCr(), fillAlpha(192), Src, color.RGBA{8, 28, 0, 192}}, + {"ycbcrNil", vgradCr(), nil, Over, color.RGBA{11, 38, 0, 255}}, + {"ycbcrNilSrc", vgradCr(), nil, Src, color.RGBA{11, 38, 0, 255}}, // Variable mask and variable source. // At (x, y) == (8, 8): // The destination pixel is {136, 0, 0, 255}. // The source pixel is {0, 0, 255, 255}. // The mask pixel's alpha is 102, or 40%. - {"generic", fillBlue(255), vgradAlpha(192), Over, image.RGBAColor{81, 0, 102, 255}}, - {"genericSrc", fillBlue(255), vgradAlpha(192), Src, image.RGBAColor{0, 0, 102, 102}}, + {"generic", fillBlue(255), vgradAlpha(192), Over, color.RGBA{81, 0, 102, 255}}, + {"genericSrc", fillBlue(255), vgradAlpha(192), Src, color.RGBA{0, 0, 102, 102}}, } func makeGolden(dst image.Image, r image.Rectangle, src image.Image, sp image.Point, mask image.Image, mp image.Point, op Op) image.Image { @@ -163,7 +164,7 @@ func makeGolden(dst image.Image, r image.Rectangle, src image.Image, sp image.Po if mask != nil { mb = mask.Bounds() } - golden := image.NewRGBA(b.Max.X, b.Max.Y) + golden := image.NewRGBA(image.Rect(0, 0, b.Max.X, b.Max.Y)) for y := r.Min.Y; y < r.Max.Y; y++ { sy := y + sp.Y - r.Min.Y my := y + mp.Y - r.Min.Y @@ -191,7 +192,7 @@ func makeGolden(dst image.Image, r image.Rectangle, src image.Image, sp image.Po _, _, _, ma = mask.At(mx, my).RGBA() } a := M - (sa * ma / M) - golden.Set(x, y, image.RGBA64Color{ + golden.Set(x, y, color.RGBA64{ uint16((dr*a + sr*ma) / M), uint16((dg*a + sg*ma) / M), uint16((db*a + sb*ma) / M), @@ -281,15 +282,15 @@ func TestDrawOverlap(t *testing.T) { // TestNonZeroSrcPt checks drawing with a non-zero src point parameter. func TestNonZeroSrcPt(t *testing.T) { - a := image.NewRGBA(1, 1) - b := image.NewRGBA(2, 2) - b.Set(0, 0, image.RGBAColor{0, 0, 0, 5}) - b.Set(1, 0, image.RGBAColor{0, 0, 5, 5}) - b.Set(0, 1, image.RGBAColor{0, 5, 0, 5}) - b.Set(1, 1, image.RGBAColor{5, 0, 0, 5}) + a := image.NewRGBA(image.Rect(0, 0, 1, 1)) + b := image.NewRGBA(image.Rect(0, 0, 2, 2)) + b.Set(0, 0, color.RGBA{0, 0, 0, 5}) + b.Set(1, 0, color.RGBA{0, 0, 5, 5}) + b.Set(0, 1, color.RGBA{0, 5, 0, 5}) + b.Set(1, 1, color.RGBA{5, 0, 0, 5}) Draw(a, image.Rect(0, 0, 1, 1), b, image.Pt(1, 1), Over) - if !eq(image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) { - t.Errorf("non-zero src pt: want %v got %v", image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) + if !eq(color.RGBA{5, 0, 0, 5}, a.At(0, 0)) { + t.Errorf("non-zero src pt: want %v got %v", color.RGBA{5, 0, 0, 5}, a.At(0, 0)) } } @@ -310,10 +311,10 @@ func TestFill(t *testing.T) { image.Rect(20, 20, 29, 29), } for _, r := range rr { - m := image.NewRGBA(40, 30).SubImage(r).(*image.RGBA) + m := image.NewRGBA(image.Rect(0, 0, 40, 30)).SubImage(r).(*image.RGBA) b := m.Bounds() - c := image.RGBAColor{11, 0, 0, 255} - src := &image.ColorImage{c} + c := color.RGBA{11, 0, 0, 255} + src := &image.Uniform{c} check := func(desc string) { for y := b.Min.Y; y < b.Max.Y; y++ { for x := b.Min.X; x < b.Max.X; x++ { @@ -332,22 +333,22 @@ func TestFill(t *testing.T) { } check("pixel") // Draw 1 row at a time. - c = image.RGBAColor{0, 22, 0, 255} - src = &image.ColorImage{c} + c = color.RGBA{0, 22, 0, 255} + src = &image.Uniform{c} for y := b.Min.Y; y < b.Max.Y; y++ { DrawMask(m, image.Rect(b.Min.X, y, b.Max.X, y+1), src, image.ZP, nil, image.ZP, Src) } check("row") // Draw 1 column at a time. - c = image.RGBAColor{0, 0, 33, 255} - src = &image.ColorImage{c} + c = color.RGBA{0, 0, 33, 255} + src = &image.Uniform{c} for x := b.Min.X; x < b.Max.X; x++ { DrawMask(m, image.Rect(x, b.Min.Y, x+1, b.Max.Y), src, image.ZP, nil, image.ZP, Src) } check("column") // Draw the whole image at once. - c = image.RGBAColor{44, 55, 66, 77} - src = &image.ColorImage{c} + c = color.RGBA{44, 55, 66, 77} + src = &image.Uniform{c} DrawMask(m, b, src, image.ZP, nil, image.ZP, Src) check("whole") } diff --git a/libgo/go/image/gif/reader.go b/libgo/go/image/gif/reader.go index e39b797..a5a4265 100644 --- a/libgo/go/image/gif/reader.go +++ b/libgo/go/image/gif/reader.go @@ -12,6 +12,7 @@ import ( "compress/lzw" "fmt" "image" + "image/color" "io" "os" ) @@ -76,7 +77,7 @@ type decoder struct { // Computed. pixelSize uint - globalColorMap image.PalettedColorModel + globalColorMap color.Palette // Used when decoding. delay []int @@ -235,7 +236,7 @@ func (d *decoder) readHeaderAndScreenDescriptor() os.Error { return nil } -func (d *decoder) readColorMap() (image.PalettedColorModel, os.Error) { +func (d *decoder) readColorMap() (color.Palette, os.Error) { if d.pixelSize > 8 { return nil, fmt.Errorf("gif: can't handle %d bits per pixel", d.pixelSize) } @@ -248,10 +249,10 @@ func (d *decoder) readColorMap() (image.PalettedColorModel, os.Error) { if err != nil { return nil, fmt.Errorf("gif: short read on color map: %s", err) } - colorMap := make(image.PalettedColorModel, numColors) + colorMap := make(color.Palette, numColors) j := 0 for i := range colorMap { - colorMap[i] = image.RGBAColor{d.tmp[j+0], d.tmp[j+1], d.tmp[j+2], 0xFF} + colorMap[i] = color.RGBA{d.tmp[j+0], d.tmp[j+1], d.tmp[j+2], 0xFF} j += 3 } return colorMap, nil @@ -319,9 +320,9 @@ func (d *decoder) readGraphicControl() os.Error { return nil } -func (d *decoder) setTransparency(colorMap image.PalettedColorModel) { +func (d *decoder) setTransparency(colorMap color.Palette) { if int(d.transparentIndex) < len(colorMap) { - colorMap[d.transparentIndex] = image.RGBAColor{} + colorMap[d.transparentIndex] = color.RGBA{} } } @@ -334,10 +335,7 @@ func (d *decoder) newImageFromDescriptor() (*image.Paletted, os.Error) { width := int(d.tmp[4]) + int(d.tmp[5])<<8 height := int(d.tmp[6]) + int(d.tmp[7])<<8 d.imageFields = d.tmp[8] - m := image.NewPaletted(width, height, nil) - // Overwrite the rectangle to take account of left and top. - m.Rect = image.Rect(left, top, left+width, top+height) - return m, nil + return image.NewPaletted(image.Rect(left, top, left+width, top+height), nil), nil } func (d *decoder) readBlock() (int, os.Error) { diff --git a/libgo/go/image/image.go b/libgo/go/image/image.go index 7c7a4b7..a0dd930 100644 --- a/libgo/go/image/image.go +++ b/libgo/go/image/image.go @@ -8,23 +8,38 @@ // http://blog.golang.org/2011/09/go-image-package.html package image +import ( + "image/color" +) + // Config holds an image's color model and dimensions. type Config struct { - ColorModel ColorModel + ColorModel color.Model Width, Height int } -// Image is a finite rectangular grid of Colors drawn from a ColorModel. +// Image is a finite rectangular grid of Colors drawn from a color model. type Image interface { - // ColorModel returns the Image's ColorModel. - ColorModel() ColorModel + // ColorModel returns the Image's color model. + ColorModel() color.Model // Bounds returns the domain for which At can return non-zero color. // The bounds do not necessarily contain the point (0, 0). Bounds() Rectangle // At returns the color of the pixel at (x, y). // At(Bounds().Min.X, Bounds().Min.Y) returns the upper-left pixel of the grid. // At(Bounds().Max.X-1, Bounds().Max.Y-1) returns the lower-right one. - At(x, y int) Color + At(x, y int) color.Color +} + +// PalettedImage is an image whose colors may come from a limited palette. +// If m is a PalettedImage and m.ColorModel() returns a PalettedColorModel p, +// then m.At(x, y) should be equivalent to p[m.ColorIndexAt(x, y)]. If m's +// color model is not a PalettedColorModel, then ColorIndexAt's behavior is +// undefined. +type PalettedImage interface { + // ColorIndexAt returns the palette index of the pixel at (x, y). + ColorIndexAt(x, y int) uint8 + Image } // RGBA is an in-memory image of RGBAColor values. @@ -38,31 +53,31 @@ type RGBA struct { Rect Rectangle } -func (p *RGBA) ColorModel() ColorModel { return RGBAColorModel } +func (p *RGBA) ColorModel() color.Model { return color.RGBAModel } func (p *RGBA) Bounds() Rectangle { return p.Rect } -func (p *RGBA) At(x, y int) Color { +func (p *RGBA) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return RGBAColor{} + return color.RGBA{} } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*4 - return RGBAColor{p.Pix[i+0], p.Pix[i+1], p.Pix[i+2], p.Pix[i+3]} + return color.RGBA{p.Pix[i+0], p.Pix[i+1], p.Pix[i+2], p.Pix[i+3]} } -func (p *RGBA) Set(x, y int, c Color) { +func (p *RGBA) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*4 - c1 := toRGBAColor(c).(RGBAColor) + c1 := color.RGBAModel.Convert(c).(color.RGBA) p.Pix[i+0] = c1.R p.Pix[i+1] = c1.G p.Pix[i+2] = c1.B p.Pix[i+3] = c1.A } -func (p *RGBA) SetRGBA(x, y int, c RGBAColor) { +func (p *RGBA) SetRGBA(x, y int, c color.RGBA) { if !(Point{x, y}.In(p.Rect)) { return } @@ -110,9 +125,10 @@ func (p *RGBA) Opaque() bool { } // NewRGBA returns a new RGBA with the given width and height. -func NewRGBA(w, h int) *RGBA { +func NewRGBA(r Rectangle) *RGBA { + w, h := r.Dx(), r.Dy() buf := make([]uint8, 4*w*h) - return &RGBA{buf, 4 * w, Rectangle{ZP, Point{w, h}}} + return &RGBA{buf, 4 * w, r} } // RGBA64 is an in-memory image of RGBA64Color values. @@ -126,16 +142,16 @@ type RGBA64 struct { Rect Rectangle } -func (p *RGBA64) ColorModel() ColorModel { return RGBA64ColorModel } +func (p *RGBA64) ColorModel() color.Model { return color.RGBA64Model } func (p *RGBA64) Bounds() Rectangle { return p.Rect } -func (p *RGBA64) At(x, y int) Color { +func (p *RGBA64) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return RGBA64Color{} + return color.RGBA64{} } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*8 - return RGBA64Color{ + return color.RGBA64{ uint16(p.Pix[i+0])<<8 | uint16(p.Pix[i+1]), uint16(p.Pix[i+2])<<8 | uint16(p.Pix[i+3]), uint16(p.Pix[i+4])<<8 | uint16(p.Pix[i+5]), @@ -143,12 +159,12 @@ func (p *RGBA64) At(x, y int) Color { } } -func (p *RGBA64) Set(x, y int, c Color) { +func (p *RGBA64) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*8 - c1 := toRGBA64Color(c).(RGBA64Color) + c1 := color.RGBA64Model.Convert(c).(color.RGBA64) p.Pix[i+0] = uint8(c1.R >> 8) p.Pix[i+1] = uint8(c1.R) p.Pix[i+2] = uint8(c1.G >> 8) @@ -159,7 +175,7 @@ func (p *RGBA64) Set(x, y int, c Color) { p.Pix[i+7] = uint8(c1.A) } -func (p *RGBA64) SetRGBA64(x, y int, c RGBA64Color) { +func (p *RGBA64) SetRGBA64(x, y int, c color.RGBA64) { if !(Point{x, y}.In(p.Rect)) { return } @@ -211,9 +227,10 @@ func (p *RGBA64) Opaque() bool { } // NewRGBA64 returns a new RGBA64 with the given width and height. -func NewRGBA64(w, h int) *RGBA64 { +func NewRGBA64(r Rectangle) *RGBA64 { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 8*w*h) - return &RGBA64{pix, 8 * w, Rectangle{ZP, Point{w, h}}} + return &RGBA64{pix, 8 * w, r} } // NRGBA is an in-memory image of NRGBAColor values. @@ -227,31 +244,31 @@ type NRGBA struct { Rect Rectangle } -func (p *NRGBA) ColorModel() ColorModel { return NRGBAColorModel } +func (p *NRGBA) ColorModel() color.Model { return color.NRGBAModel } func (p *NRGBA) Bounds() Rectangle { return p.Rect } -func (p *NRGBA) At(x, y int) Color { +func (p *NRGBA) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return NRGBAColor{} + return color.NRGBA{} } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*4 - return NRGBAColor{p.Pix[i+0], p.Pix[i+1], p.Pix[i+2], p.Pix[i+3]} + return color.NRGBA{p.Pix[i+0], p.Pix[i+1], p.Pix[i+2], p.Pix[i+3]} } -func (p *NRGBA) Set(x, y int, c Color) { +func (p *NRGBA) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*4 - c1 := toNRGBAColor(c).(NRGBAColor) + c1 := color.NRGBAModel.Convert(c).(color.NRGBA) p.Pix[i+0] = c1.R p.Pix[i+1] = c1.G p.Pix[i+2] = c1.B p.Pix[i+3] = c1.A } -func (p *NRGBA) SetNRGBA(x, y int, c NRGBAColor) { +func (p *NRGBA) SetNRGBA(x, y int, c color.NRGBA) { if !(Point{x, y}.In(p.Rect)) { return } @@ -299,9 +316,10 @@ func (p *NRGBA) Opaque() bool { } // NewNRGBA returns a new NRGBA with the given width and height. -func NewNRGBA(w, h int) *NRGBA { +func NewNRGBA(r Rectangle) *NRGBA { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 4*w*h) - return &NRGBA{pix, 4 * w, Rectangle{ZP, Point{w, h}}} + return &NRGBA{pix, 4 * w, r} } // NRGBA64 is an in-memory image of NRGBA64Color values. @@ -315,16 +333,16 @@ type NRGBA64 struct { Rect Rectangle } -func (p *NRGBA64) ColorModel() ColorModel { return NRGBA64ColorModel } +func (p *NRGBA64) ColorModel() color.Model { return color.NRGBA64Model } func (p *NRGBA64) Bounds() Rectangle { return p.Rect } -func (p *NRGBA64) At(x, y int) Color { +func (p *NRGBA64) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return NRGBA64Color{} + return color.NRGBA64{} } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*8 - return NRGBA64Color{ + return color.NRGBA64{ uint16(p.Pix[i+0])<<8 | uint16(p.Pix[i+1]), uint16(p.Pix[i+2])<<8 | uint16(p.Pix[i+3]), uint16(p.Pix[i+4])<<8 | uint16(p.Pix[i+5]), @@ -332,12 +350,12 @@ func (p *NRGBA64) At(x, y int) Color { } } -func (p *NRGBA64) Set(x, y int, c Color) { +func (p *NRGBA64) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*8 - c1 := toNRGBA64Color(c).(NRGBA64Color) + c1 := color.NRGBA64Model.Convert(c).(color.NRGBA64) p.Pix[i+0] = uint8(c1.R >> 8) p.Pix[i+1] = uint8(c1.R) p.Pix[i+2] = uint8(c1.G >> 8) @@ -348,7 +366,7 @@ func (p *NRGBA64) Set(x, y int, c Color) { p.Pix[i+7] = uint8(c1.A) } -func (p *NRGBA64) SetNRGBA64(x, y int, c NRGBA64Color) { +func (p *NRGBA64) SetNRGBA64(x, y int, c color.NRGBA64) { if !(Point{x, y}.In(p.Rect)) { return } @@ -400,9 +418,10 @@ func (p *NRGBA64) Opaque() bool { } // NewNRGBA64 returns a new NRGBA64 with the given width and height. -func NewNRGBA64(w, h int) *NRGBA64 { +func NewNRGBA64(r Rectangle) *NRGBA64 { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 8*w*h) - return &NRGBA64{pix, 8 * w, Rectangle{ZP, Point{w, h}}} + return &NRGBA64{pix, 8 * w, r} } // Alpha is an in-memory image of AlphaColor values. @@ -416,27 +435,27 @@ type Alpha struct { Rect Rectangle } -func (p *Alpha) ColorModel() ColorModel { return AlphaColorModel } +func (p *Alpha) ColorModel() color.Model { return color.AlphaModel } func (p *Alpha) Bounds() Rectangle { return p.Rect } -func (p *Alpha) At(x, y int) Color { +func (p *Alpha) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return AlphaColor{} + return color.Alpha{} } i := (y-p.Rect.Min.Y)*p.Stride + (x - p.Rect.Min.X) - return AlphaColor{p.Pix[i]} + return color.Alpha{p.Pix[i]} } -func (p *Alpha) Set(x, y int, c Color) { +func (p *Alpha) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x - p.Rect.Min.X) - p.Pix[i] = toAlphaColor(c).(AlphaColor).A + p.Pix[i] = color.AlphaModel.Convert(c).(color.Alpha).A } -func (p *Alpha) SetAlpha(x, y int, c AlphaColor) { +func (p *Alpha) SetAlpha(x, y int, c color.Alpha) { if !(Point{x, y}.In(p.Rect)) { return } @@ -481,9 +500,10 @@ func (p *Alpha) Opaque() bool { } // NewAlpha returns a new Alpha with the given width and height. -func NewAlpha(w, h int) *Alpha { +func NewAlpha(r Rectangle) *Alpha { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 1*w*h) - return &Alpha{pix, 1 * w, Rectangle{ZP, Point{w, h}}} + return &Alpha{pix, 1 * w, r} } // Alpha16 is an in-memory image of Alpha16Color values. @@ -497,29 +517,29 @@ type Alpha16 struct { Rect Rectangle } -func (p *Alpha16) ColorModel() ColorModel { return Alpha16ColorModel } +func (p *Alpha16) ColorModel() color.Model { return color.Alpha16Model } func (p *Alpha16) Bounds() Rectangle { return p.Rect } -func (p *Alpha16) At(x, y int) Color { +func (p *Alpha16) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return Alpha16Color{} + return color.Alpha16{} } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*2 - return Alpha16Color{uint16(p.Pix[i+0])<<8 | uint16(p.Pix[i+1])} + return color.Alpha16{uint16(p.Pix[i+0])<<8 | uint16(p.Pix[i+1])} } -func (p *Alpha16) Set(x, y int, c Color) { +func (p *Alpha16) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*2 - c1 := toAlpha16Color(c).(Alpha16Color) + c1 := color.Alpha16Model.Convert(c).(color.Alpha16) p.Pix[i+0] = uint8(c1.A >> 8) p.Pix[i+1] = uint8(c1.A) } -func (p *Alpha16) SetAlpha16(x, y int, c Alpha16Color) { +func (p *Alpha16) SetAlpha16(x, y int, c color.Alpha16) { if !(Point{x, y}.In(p.Rect)) { return } @@ -565,9 +585,10 @@ func (p *Alpha16) Opaque() bool { } // NewAlpha16 returns a new Alpha16 with the given width and height. -func NewAlpha16(w, h int) *Alpha16 { +func NewAlpha16(r Rectangle) *Alpha16 { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 2*w*h) - return &Alpha16{pix, 2 * w, Rectangle{ZP, Point{w, h}}} + return &Alpha16{pix, 2 * w, r} } // Gray is an in-memory image of GrayColor values. @@ -581,27 +602,27 @@ type Gray struct { Rect Rectangle } -func (p *Gray) ColorModel() ColorModel { return GrayColorModel } +func (p *Gray) ColorModel() color.Model { return color.GrayModel } func (p *Gray) Bounds() Rectangle { return p.Rect } -func (p *Gray) At(x, y int) Color { +func (p *Gray) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return GrayColor{} + return color.Gray{} } i := (y-p.Rect.Min.Y)*p.Stride + (x - p.Rect.Min.X) - return GrayColor{p.Pix[i]} + return color.Gray{p.Pix[i]} } -func (p *Gray) Set(x, y int, c Color) { +func (p *Gray) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x - p.Rect.Min.X) - p.Pix[i] = toGrayColor(c).(GrayColor).Y + p.Pix[i] = color.GrayModel.Convert(c).(color.Gray).Y } -func (p *Gray) SetGray(x, y int, c GrayColor) { +func (p *Gray) SetGray(x, y int, c color.Gray) { if !(Point{x, y}.In(p.Rect)) { return } @@ -633,9 +654,10 @@ func (p *Gray) Opaque() bool { } // NewGray returns a new Gray with the given width and height. -func NewGray(w, h int) *Gray { +func NewGray(r Rectangle) *Gray { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 1*w*h) - return &Gray{pix, 1 * w, Rectangle{ZP, Point{w, h}}} + return &Gray{pix, 1 * w, r} } // Gray16 is an in-memory image of Gray16Color values. @@ -649,29 +671,29 @@ type Gray16 struct { Rect Rectangle } -func (p *Gray16) ColorModel() ColorModel { return Gray16ColorModel } +func (p *Gray16) ColorModel() color.Model { return color.Gray16Model } func (p *Gray16) Bounds() Rectangle { return p.Rect } -func (p *Gray16) At(x, y int) Color { +func (p *Gray16) At(x, y int) color.Color { if !(Point{x, y}.In(p.Rect)) { - return Gray16Color{} + return color.Gray16{} } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*2 - return Gray16Color{uint16(p.Pix[i+0])<<8 | uint16(p.Pix[i+1])} + return color.Gray16{uint16(p.Pix[i+0])<<8 | uint16(p.Pix[i+1])} } -func (p *Gray16) Set(x, y int, c Color) { +func (p *Gray16) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } i := (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*2 - c1 := toGray16Color(c).(Gray16Color) + c1 := color.Gray16Model.Convert(c).(color.Gray16) p.Pix[i+0] = uint8(c1.Y >> 8) p.Pix[i+1] = uint8(c1.Y) } -func (p *Gray16) SetGray16(x, y int, c Gray16Color) { +func (p *Gray16) SetGray16(x, y int, c color.Gray16) { if !(Point{x, y}.In(p.Rect)) { return } @@ -704,50 +726,10 @@ func (p *Gray16) Opaque() bool { } // NewGray16 returns a new Gray16 with the given width and height. -func NewGray16(w, h int) *Gray16 { +func NewGray16(r Rectangle) *Gray16 { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 2*w*h) - return &Gray16{pix, 2 * w, Rectangle{ZP, Point{w, h}}} -} - -// A PalettedColorModel represents a fixed palette of at most 256 colors. -type PalettedColorModel []Color - -func diff(a, b uint32) uint32 { - if a > b { - return a - b - } - return b - a -} - -// Convert returns the palette color closest to c in Euclidean R,G,B space. -func (p PalettedColorModel) Convert(c Color) Color { - if len(p) == 0 { - return nil - } - return p[p.Index(c)] -} - -// Index returns the index of the palette color closest to c in Euclidean -// R,G,B space. -func (p PalettedColorModel) Index(c Color) int { - cr, cg, cb, _ := c.RGBA() - // Shift by 1 bit to avoid potential uint32 overflow in sum-squared-difference. - cr >>= 1 - cg >>= 1 - cb >>= 1 - ret, bestSSD := 0, uint32(1<<32-1) - for i, v := range p { - vr, vg, vb, _ := v.RGBA() - vr >>= 1 - vg >>= 1 - vb >>= 1 - dr, dg, db := diff(cr, vr), diff(cg, vg), diff(cb, vb) - ssd := (dr * dr) + (dg * dg) + (db * db) - if ssd < bestSSD { - ret, bestSSD = i, ssd - } - } - return ret + return &Gray16{pix, 2 * w, r} } // Paletted is an in-memory image of uint8 indices into a given palette. @@ -760,14 +742,14 @@ type Paletted struct { // Rect is the image's bounds. Rect Rectangle // Palette is the image's palette. - Palette PalettedColorModel + Palette color.Palette } -func (p *Paletted) ColorModel() ColorModel { return p.Palette } +func (p *Paletted) ColorModel() color.Model { return p.Palette } func (p *Paletted) Bounds() Rectangle { return p.Rect } -func (p *Paletted) At(x, y int) Color { +func (p *Paletted) At(x, y int) color.Color { if len(p.Palette) == 0 { return nil } @@ -778,7 +760,7 @@ func (p *Paletted) At(x, y int) Color { return p.Palette[p.Pix[i]] } -func (p *Paletted) Set(x, y int, c Color) { +func (p *Paletted) Set(x, y int, c color.Color) { if !(Point{x, y}.In(p.Rect)) { return } @@ -847,7 +829,8 @@ func (p *Paletted) Opaque() bool { } // NewPaletted returns a new Paletted with the given width, height and palette. -func NewPaletted(w, h int, m PalettedColorModel) *Paletted { +func NewPaletted(r Rectangle, p color.Palette) *Paletted { + w, h := r.Dx(), r.Dy() pix := make([]uint8, 1*w*h) - return &Paletted{pix, 1 * w, Rectangle{ZP, Point{w, h}}, m} + return &Paletted{pix, 1 * w, r, p} } diff --git a/libgo/go/image/image_test.go b/libgo/go/image/image_test.go index 9519acf..2b3f149 100644 --- a/libgo/go/image/image_test.go +++ b/libgo/go/image/image_test.go @@ -6,17 +6,18 @@ package image_test import ( . "image" + "image/color" "testing" ) type image interface { Image Opaque() bool - Set(int, int, Color) + Set(int, int, color.Color) SubImage(Rectangle) Image } -func cmp(t *testing.T, cm ColorModel, c0, c1 Color) bool { +func cmp(t *testing.T, cm color.Model, c0, c1 color.Color) bool { r0, g0, b0, a0 := cm.Convert(c0).RGBA() r1, g1, b1, a1 := cm.Convert(c1).RGBA() return r0 == r1 && g0 == g1 && b0 == b1 && a0 == a1 @@ -24,15 +25,15 @@ func cmp(t *testing.T, cm ColorModel, c0, c1 Color) bool { func TestImage(t *testing.T) { testImage := []image{ - NewRGBA(10, 10), - NewRGBA64(10, 10), - NewNRGBA(10, 10), - NewNRGBA64(10, 10), - NewAlpha(10, 10), - NewAlpha16(10, 10), - NewGray(10, 10), - NewGray16(10, 10), - NewPaletted(10, 10, PalettedColorModel{ + NewRGBA(Rect(0, 0, 10, 10)), + NewRGBA64(Rect(0, 0, 10, 10)), + NewNRGBA(Rect(0, 0, 10, 10)), + NewNRGBA64(Rect(0, 0, 10, 10)), + NewAlpha(Rect(0, 0, 10, 10)), + NewAlpha16(Rect(0, 0, 10, 10)), + NewGray(Rect(0, 0, 10, 10)), + NewGray16(Rect(0, 0, 10, 10)), + NewPaletted(Rect(0, 0, 10, 10), color.Palette{ Transparent, Opaque, }), @@ -82,14 +83,14 @@ func TestImage(t *testing.T) { } func Test16BitsPerColorChannel(t *testing.T) { - testColorModel := []ColorModel{ - RGBA64ColorModel, - NRGBA64ColorModel, - Alpha16ColorModel, - Gray16ColorModel, + testColorModel := []color.Model{ + color.RGBA64Model, + color.NRGBA64Model, + color.Alpha16Model, + color.Gray16Model, } for _, cm := range testColorModel { - c := cm.Convert(RGBA64Color{0x1234, 0x1234, 0x1234, 0x1234}) // Premultiplied alpha. + c := cm.Convert(color.RGBA64{0x1234, 0x1234, 0x1234, 0x1234}) // Premultiplied alpha. r, _, _, _ := c.RGBA() if r != 0x1234 { t.Errorf("%T: want red value 0x%04x got 0x%04x", c, 0x1234, r) @@ -97,13 +98,13 @@ func Test16BitsPerColorChannel(t *testing.T) { } } testImage := []image{ - NewRGBA64(10, 10), - NewNRGBA64(10, 10), - NewAlpha16(10, 10), - NewGray16(10, 10), + NewRGBA64(Rect(0, 0, 10, 10)), + NewNRGBA64(Rect(0, 0, 10, 10)), + NewAlpha16(Rect(0, 0, 10, 10)), + NewGray16(Rect(0, 0, 10, 10)), } for _, m := range testImage { - m.Set(1, 2, NRGBA64Color{0xffff, 0xffff, 0xffff, 0x1357}) // Non-premultiplied alpha. + m.Set(1, 2, color.NRGBA64{0xffff, 0xffff, 0xffff, 0x1357}) // Non-premultiplied alpha. r, _, _, _ := m.At(1, 2).RGBA() if r != 0x1357 { t.Errorf("%T: want red value 0x%04x got 0x%04x", m, 0x1357, r) diff --git a/libgo/go/image/jpeg/reader.go b/libgo/go/image/jpeg/reader.go index 3f22c52..450355e 100644 --- a/libgo/go/image/jpeg/reader.go +++ b/libgo/go/image/jpeg/reader.go @@ -10,6 +10,7 @@ package jpeg import ( "bufio" "image" + "image/color" "image/ycbcr" "io" "os" @@ -199,7 +200,7 @@ func (d *decoder) processDQT(n int) os.Error { // makeImg allocates and initializes the destination image. func (d *decoder) makeImg(h0, v0, mxx, myy int) { if d.nComp == nGrayComponent { - m := image.NewGray(8*mxx, 8*myy) + m := image.NewGray(image.Rect(0, 0, 8*mxx, 8*myy)) d.img1 = m.SubImage(image.Rect(0, 0, d.width, d.height)).(*image.Gray) return } @@ -464,7 +465,7 @@ func DecodeConfig(r io.Reader) (image.Config, os.Error) { } switch d.nComp { case nGrayComponent: - return image.Config{image.GrayColorModel, d.width, d.height}, nil + return image.Config{color.GrayModel, d.width, d.height}, nil case nColorComponent: return image.Config{ycbcr.YCbCrColorModel, d.width, d.height}, nil } diff --git a/libgo/go/image/jpeg/writer_test.go b/libgo/go/image/jpeg/writer_test.go index 7aec70f..0378252 100644 --- a/libgo/go/image/jpeg/writer_test.go +++ b/libgo/go/image/jpeg/writer_test.go @@ -7,6 +7,7 @@ package jpeg import ( "bytes" "image" + "image/color" "image/png" "io/ioutil" "rand" @@ -90,13 +91,13 @@ func TestWriter(t *testing.T) { func BenchmarkEncodeRGBOpaque(b *testing.B) { b.StopTimer() - img := image.NewRGBA(640, 480) + img := image.NewRGBA(image.Rect(0, 0, 640, 480)) // Set all pixels to 0xFF alpha to force opaque mode. bo := img.Bounds() rnd := rand.New(rand.NewSource(123)) for y := bo.Min.Y; y < bo.Max.Y; y++ { for x := bo.Min.X; x < bo.Max.X; x++ { - img.Set(x, y, image.RGBAColor{ + img.Set(x, y, color.RGBA{ uint8(rnd.Intn(256)), uint8(rnd.Intn(256)), uint8(rnd.Intn(256)), diff --git a/libgo/go/image/names.go b/libgo/go/image/names.go index c309684..a7ad51d 100644 --- a/libgo/go/image/names.go +++ b/libgo/go/image/names.go @@ -4,43 +4,47 @@ package image +import ( + "image/color" +) + var ( - // Black is an opaque black ColorImage. - Black = NewColorImage(Gray16Color{0}) - // White is an opaque white ColorImage. - White = NewColorImage(Gray16Color{0xffff}) - // Transparent is a fully transparent ColorImage. - Transparent = NewColorImage(Alpha16Color{0}) - // Opaque is a fully opaque ColorImage. - Opaque = NewColorImage(Alpha16Color{0xffff}) + // Black is an opaque black uniform image. + Black = NewUniform(color.Black) + // White is an opaque white uniform image. + White = NewUniform(color.White) + // Transparent is a fully transparent uniform image. + Transparent = NewUniform(color.Transparent) + // Opaque is a fully opaque uniform image. + Opaque = NewUniform(color.Opaque) ) -// A ColorImage is an infinite-sized Image of uniform Color. -// It implements both the Color and Image interfaces. -type ColorImage struct { - C Color +// Uniform is an infinite-sized Image of uniform color. +// It implements both the color.Color and Image interfaces. +type Uniform struct { + C color.Color } -func (c *ColorImage) RGBA() (r, g, b, a uint32) { +func (c *Uniform) RGBA() (r, g, b, a uint32) { return c.C.RGBA() } -func (c *ColorImage) ColorModel() ColorModel { - return ColorModelFunc(func(Color) Color { return c.C }) +func (c *Uniform) ColorModel() color.Model { + return color.ModelFunc(func(color.Color) color.Color { return c.C }) } -func (c *ColorImage) Bounds() Rectangle { return Rectangle{Point{-1e9, -1e9}, Point{1e9, 1e9}} } +func (c *Uniform) Bounds() Rectangle { return Rectangle{Point{-1e9, -1e9}, Point{1e9, 1e9}} } -func (c *ColorImage) At(x, y int) Color { return c.C } +func (c *Uniform) At(x, y int) color.Color { return c.C } // Opaque scans the entire image and returns whether or not it is fully opaque. -func (c *ColorImage) Opaque() bool { +func (c *Uniform) Opaque() bool { _, _, _, a := c.C.RGBA() return a == 0xffff } -func NewColorImage(c Color) *ColorImage { - return &ColorImage{c} +func NewUniform(c color.Color) *Uniform { + return &Uniform{c} } // A Tiled is an infinite-sized Image that repeats another Image in both @@ -51,13 +55,13 @@ type Tiled struct { Offset Point } -func (t *Tiled) ColorModel() ColorModel { +func (t *Tiled) ColorModel() color.Model { return t.I.ColorModel() } func (t *Tiled) Bounds() Rectangle { return Rectangle{Point{-1e9, -1e9}, Point{1e9, 1e9}} } -func (t *Tiled) At(x, y int) Color { +func (t *Tiled) At(x, y int) color.Color { p := Point{x, y}.Add(t.Offset).Mod(t.I.Bounds()) return t.I.At(p.X, p.Y) } diff --git a/libgo/go/image/png/reader.go b/libgo/go/image/png/reader.go index 8c76afa..66f1916 100644 --- a/libgo/go/image/png/reader.go +++ b/libgo/go/image/png/reader.go @@ -4,15 +4,17 @@ // Package png implements a PNG image decoder and encoder. // -// The PNG specification is at http://www.libpng.org/pub/png/spec/1.2/PNG-Contents.html +// The PNG specification is at http://www.w3.org/TR/PNG/. package png import ( "compress/zlib" + "encoding/binary" "fmt" "hash" "hash/crc32" "image" + "image/color" "io" "os" ) @@ -61,6 +63,7 @@ const ( // chunks must appear in that order. There may be multiple IDAT chunks, and // IDAT chunks must be sequential (i.e. they may not have any other chunks // between them). +// http://www.w3.org/TR/PNG/#5ChunkOrdering const ( dsStart = iota dsSeenIHDR @@ -71,19 +74,16 @@ const ( const pngHeader = "\x89PNG\r\n\x1a\n" -type imgOrErr struct { - img image.Image - err os.Error -} - type decoder struct { + r io.Reader + img image.Image + crc hash.Hash32 width, height int depth int - palette image.PalettedColorModel + palette color.Palette cb int stage int - idatWriter io.WriteCloser - idatDone chan imgOrErr + idatLength uint32 tmp [3 * 256]byte } @@ -94,23 +94,11 @@ func (e FormatError) String() string { return "png: invalid format: " + string(e var chunkOrderError = FormatError("chunk out of order") -// An IDATDecodingError wraps an inner error (such as a ZLIB decoding error) encountered while processing an IDAT chunk. -type IDATDecodingError struct { - Err os.Error -} - -func (e IDATDecodingError) String() string { return "png: IDAT decoding error: " + e.Err.String() } - // An UnsupportedError reports that the input uses a valid but unimplemented PNG feature. type UnsupportedError string func (e UnsupportedError) String() string { return "png: unsupported feature: " + string(e) } -// Big-endian. -func parseUint32(b []uint8) uint32 { - return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) -} - func abs(x int) int { if x < 0 { return -x @@ -125,20 +113,19 @@ func min(a, b int) int { return b } -func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Error { +func (d *decoder) parseIHDR(length uint32) os.Error { if length != 13 { return FormatError("bad IHDR length") } - _, err := io.ReadFull(r, d.tmp[0:13]) - if err != nil { + if _, err := io.ReadFull(d.r, d.tmp[:13]); err != nil { return err } - crc.Write(d.tmp[0:13]) + d.crc.Write(d.tmp[:13]) if d.tmp[10] != 0 || d.tmp[11] != 0 || d.tmp[12] != 0 { return UnsupportedError("compression, filter or interlace method") } - w := int32(parseUint32(d.tmp[0:4])) - h := int32(parseUint32(d.tmp[4:8])) + w := int32(binary.BigEndian.Uint32(d.tmp[0:4])) + h := int32(binary.BigEndian.Uint32(d.tmp[4:8])) if w < 0 || h < 0 { return FormatError("negative dimension") } @@ -199,24 +186,24 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro return UnsupportedError(fmt.Sprintf("bit depth %d, color type %d", d.tmp[8], d.tmp[9])) } d.width, d.height = int(w), int(h) - return nil + return d.verifyChecksum() } -func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Error { +func (d *decoder) parsePLTE(length uint32) os.Error { np := int(length / 3) // The number of palette entries. if length%3 != 0 || np <= 0 || np > 256 || np > 1<<uint(d.depth) { return FormatError("bad PLTE length") } - n, err := io.ReadFull(r, d.tmp[0:3*np]) + n, err := io.ReadFull(d.r, d.tmp[:3*np]) if err != nil { return err } - crc.Write(d.tmp[0:n]) + d.crc.Write(d.tmp[:n]) switch d.cb { case cbP1, cbP2, cbP4, cbP8: - d.palette = image.PalettedColorModel(make([]image.Color, np)) + d.palette = color.Palette(make([]color.Color, np)) for i := 0; i < np; i++ { - d.palette[i] = image.RGBAColor{d.tmp[3*i+0], d.tmp[3*i+1], d.tmp[3*i+2], 0xff} + d.palette[i] = color.RGBA{d.tmp[3*i+0], d.tmp[3*i+1], d.tmp[3*i+2], 0xff} } case cbTC8, cbTCA8, cbTC16, cbTCA16: // As per the PNG spec, a PLTE chunk is optional (and for practical purposes, @@ -224,18 +211,18 @@ func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Erro default: return FormatError("PLTE, color type mismatch") } - return nil + return d.verifyChecksum() } -func (d *decoder) parsetRNS(r io.Reader, crc hash.Hash32, length uint32) os.Error { +func (d *decoder) parsetRNS(length uint32) os.Error { if length > 256 { return FormatError("bad tRNS length") } - n, err := io.ReadFull(r, d.tmp[0:length]) + n, err := io.ReadFull(d.r, d.tmp[:length]) if err != nil { return err } - crc.Write(d.tmp[0:n]) + d.crc.Write(d.tmp[:n]) switch d.cb { case cbG8, cbG16: return UnsupportedError("grayscale transparency") @@ -246,13 +233,13 @@ func (d *decoder) parsetRNS(r io.Reader, crc hash.Hash32, length uint32) os.Erro return FormatError("bad tRNS length") } for i := 0; i < n; i++ { - rgba := d.palette[i].(image.RGBAColor) - d.palette[i] = image.RGBAColor{rgba.R, rgba.G, rgba.B, d.tmp[i]} + rgba := d.palette[i].(color.RGBA) + d.palette[i] = color.RGBA{rgba.R, rgba.G, rgba.B, d.tmp[i]} } case cbGA8, cbGA16, cbTCA8, cbTCA16: return FormatError("tRNS, color type mismatch") } - return nil + return d.verifyChecksum() } // The Paeth filter function, as per the PNG specification. @@ -269,8 +256,46 @@ func paeth(a, b, c uint8) uint8 { return c } -func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { - r, err := zlib.NewReader(idat) +// Read presents one or more IDAT chunks as one continuous stream (minus the +// intermediate chunk headers and footers). If the PNG data looked like: +// ... len0 IDAT xxx crc0 len1 IDAT yy crc1 len2 IEND crc2 +// then this reader presents xxxyy. For well-formed PNG data, the decoder state +// immediately before the first Read call is that d.r is positioned between the +// first IDAT and xxx, and the decoder state immediately after the last Read +// call is that d.r is positioned between yy and crc1. +func (d *decoder) Read(p []byte) (int, os.Error) { + if len(p) == 0 { + return 0, nil + } + for d.idatLength == 0 { + // We have exhausted an IDAT chunk. Verify the checksum of that chunk. + if err := d.verifyChecksum(); err != nil { + return 0, err + } + // Read the length and chunk type of the next chunk, and check that + // it is an IDAT chunk. + if _, err := io.ReadFull(d.r, d.tmp[:8]); err != nil { + return 0, err + } + d.idatLength = binary.BigEndian.Uint32(d.tmp[:4]) + if string(d.tmp[4:8]) != "IDAT" { + return 0, FormatError("not enough pixel data") + } + d.crc.Reset() + d.crc.Write(d.tmp[4:8]) + } + if int(d.idatLength) < 0 { + return 0, UnsupportedError("IDAT chunk length overflow") + } + n, err := d.r.Read(p[:min(len(p), int(d.idatLength))]) + d.crc.Write(p[:n]) + d.idatLength -= uint32(n) + return n, err +} + +// decode decodes the IDAT data into an image. +func (d *decoder) decode() (image.Image, os.Error) { + r, err := zlib.NewReader(d) if err != nil { return nil, err } @@ -290,40 +315,40 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { switch d.cb { case cbG1, cbG2, cbG4, cbG8: bitsPerPixel = d.depth - gray = image.NewGray(d.width, d.height) + gray = image.NewGray(image.Rect(0, 0, d.width, d.height)) img = gray case cbGA8: bitsPerPixel = 16 - nrgba = image.NewNRGBA(d.width, d.height) + nrgba = image.NewNRGBA(image.Rect(0, 0, d.width, d.height)) img = nrgba case cbTC8: bitsPerPixel = 24 - rgba = image.NewRGBA(d.width, d.height) + rgba = image.NewRGBA(image.Rect(0, 0, d.width, d.height)) img = rgba case cbP1, cbP2, cbP4, cbP8: bitsPerPixel = d.depth - paletted = image.NewPaletted(d.width, d.height, d.palette) + paletted = image.NewPaletted(image.Rect(0, 0, d.width, d.height), d.palette) img = paletted maxPalette = uint8(len(d.palette) - 1) case cbTCA8: bitsPerPixel = 32 - nrgba = image.NewNRGBA(d.width, d.height) + nrgba = image.NewNRGBA(image.Rect(0, 0, d.width, d.height)) img = nrgba case cbG16: bitsPerPixel = 16 - gray16 = image.NewGray16(d.width, d.height) + gray16 = image.NewGray16(image.Rect(0, 0, d.width, d.height)) img = gray16 case cbGA16: bitsPerPixel = 32 - nrgba64 = image.NewNRGBA64(d.width, d.height) + nrgba64 = image.NewNRGBA64(image.Rect(0, 0, d.width, d.height)) img = nrgba64 case cbTC16: bitsPerPixel = 48 - rgba64 = image.NewRGBA64(d.width, d.height) + rgba64 = image.NewRGBA64(image.Rect(0, 0, d.width, d.height)) img = rgba64 case cbTCA16: bitsPerPixel = 64 - nrgba64 = image.NewNRGBA64(d.width, d.height) + nrgba64 = image.NewNRGBA64(image.Rect(0, 0, d.width, d.height)) img = nrgba64 } bytesPerPixel := (bitsPerPixel + 7) / 8 @@ -378,7 +403,7 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { for x := 0; x < d.width; x += 8 { b := cdat[x/8] for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ { - gray.SetGray(x+x2, y, image.GrayColor{(b >> 7) * 0xff}) + gray.SetGray(x+x2, y, color.Gray{(b >> 7) * 0xff}) b <<= 1 } } @@ -386,7 +411,7 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { for x := 0; x < d.width; x += 4 { b := cdat[x/4] for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ { - gray.SetGray(x+x2, y, image.GrayColor{(b >> 6) * 0x55}) + gray.SetGray(x+x2, y, color.Gray{(b >> 6) * 0x55}) b <<= 2 } } @@ -394,22 +419,22 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { for x := 0; x < d.width; x += 2 { b := cdat[x/2] for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ { - gray.SetGray(x+x2, y, image.GrayColor{(b >> 4) * 0x11}) + gray.SetGray(x+x2, y, color.Gray{(b >> 4) * 0x11}) b <<= 4 } } case cbG8: for x := 0; x < d.width; x++ { - gray.SetGray(x, y, image.GrayColor{cdat[x]}) + gray.SetGray(x, y, color.Gray{cdat[x]}) } case cbGA8: for x := 0; x < d.width; x++ { ycol := cdat[2*x+0] - nrgba.SetNRGBA(x, y, image.NRGBAColor{ycol, ycol, ycol, cdat[2*x+1]}) + nrgba.SetNRGBA(x, y, color.NRGBA{ycol, ycol, ycol, cdat[2*x+1]}) } case cbTC8: for x := 0; x < d.width; x++ { - rgba.SetRGBA(x, y, image.RGBAColor{cdat[3*x+0], cdat[3*x+1], cdat[3*x+2], 0xff}) + rgba.SetRGBA(x, y, color.RGBA{cdat[3*x+0], cdat[3*x+1], cdat[3*x+2], 0xff}) } case cbP1: for x := 0; x < d.width; x += 8 { @@ -456,25 +481,25 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { } case cbTCA8: for x := 0; x < d.width; x++ { - nrgba.SetNRGBA(x, y, image.NRGBAColor{cdat[4*x+0], cdat[4*x+1], cdat[4*x+2], cdat[4*x+3]}) + nrgba.SetNRGBA(x, y, color.NRGBA{cdat[4*x+0], cdat[4*x+1], cdat[4*x+2], cdat[4*x+3]}) } case cbG16: for x := 0; x < d.width; x++ { ycol := uint16(cdat[2*x+0])<<8 | uint16(cdat[2*x+1]) - gray16.SetGray16(x, y, image.Gray16Color{ycol}) + gray16.SetGray16(x, y, color.Gray16{ycol}) } case cbGA16: for x := 0; x < d.width; x++ { ycol := uint16(cdat[4*x+0])<<8 | uint16(cdat[4*x+1]) acol := uint16(cdat[4*x+2])<<8 | uint16(cdat[4*x+3]) - nrgba64.SetNRGBA64(x, y, image.NRGBA64Color{ycol, ycol, ycol, acol}) + nrgba64.SetNRGBA64(x, y, color.NRGBA64{ycol, ycol, ycol, acol}) } case cbTC16: for x := 0; x < d.width; x++ { rcol := uint16(cdat[6*x+0])<<8 | uint16(cdat[6*x+1]) gcol := uint16(cdat[6*x+2])<<8 | uint16(cdat[6*x+3]) bcol := uint16(cdat[6*x+4])<<8 | uint16(cdat[6*x+5]) - rgba64.SetRGBA64(x, y, image.RGBA64Color{rcol, gcol, bcol, 0xffff}) + rgba64.SetRGBA64(x, y, color.RGBA64{rcol, gcol, bcol, 0xffff}) } case cbTCA16: for x := 0; x < d.width; x++ { @@ -482,150 +507,113 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { gcol := uint16(cdat[8*x+2])<<8 | uint16(cdat[8*x+3]) bcol := uint16(cdat[8*x+4])<<8 | uint16(cdat[8*x+5]) acol := uint16(cdat[8*x+6])<<8 | uint16(cdat[8*x+7]) - nrgba64.SetNRGBA64(x, y, image.NRGBA64Color{rcol, gcol, bcol, acol}) + nrgba64.SetNRGBA64(x, y, color.NRGBA64{rcol, gcol, bcol, acol}) } } // The current row for y is the previous row for y+1. pr, cr = cr, pr } - return img, nil -} -func (d *decoder) parseIDAT(r io.Reader, crc hash.Hash32, length uint32) os.Error { - // There may be more than one IDAT chunk, but their contents must be - // treated as if it was one continuous stream (to the zlib decoder). - // We bring up an io.Pipe and write the IDAT chunks into the pipe as - // we see them, and decode the stream in a separate go-routine, which - // signals its completion (successful or not) via a channel. - if d.idatWriter == nil { - pr, pw := io.Pipe() - d.idatWriter = pw - d.idatDone = make(chan imgOrErr) - go func() { - img, err := d.idatReader(pr) - if err == os.EOF { - err = FormatError("too little IDAT") - } - pr.CloseWithError(FormatError("too much IDAT")) - d.idatDone <- imgOrErr{img, err} - }() + // Check for EOF, to verify the zlib checksum. + n, err := r.Read(pr[:1]) + if err != os.EOF { + return nil, FormatError(err.String()) } - var buf [4096]byte - for length > 0 { - n, err1 := r.Read(buf[0:min(len(buf), int(length))]) - // We delay checking err1. It is possible to get n bytes and an error, - // but if the n bytes themselves contain a FormatError, for example, we - // want to report that error, and not the one that made the Read stop. - n, err2 := d.idatWriter.Write(buf[0:n]) - if err2 != nil { - return err2 - } - if err1 != nil { - return err1 - } - crc.Write(buf[0:n]) - length -= uint32(n) + if n != 0 || d.idatLength != 0 { + return nil, FormatError("too much pixel data") } - return nil -} -func (d *decoder) parseIEND(r io.Reader, crc hash.Hash32, length uint32) os.Error { - if length != 0 { - return FormatError("bad IEND length") - } - return nil + return img, nil } -func (d *decoder) parseChunk(r io.Reader) os.Error { - // Read the length. - n, err := io.ReadFull(r, d.tmp[0:4]) - if err == os.EOF { - return io.ErrUnexpectedEOF - } +func (d *decoder) parseIDAT(length uint32) (err os.Error) { + d.idatLength = length + d.img, err = d.decode() if err != nil { return err } - length := parseUint32(d.tmp[0:4]) + return d.verifyChecksum() +} - // Read the chunk type. - n, err = io.ReadFull(r, d.tmp[0:4]) - if err == os.EOF { - return io.ErrUnexpectedEOF +func (d *decoder) parseIEND(length uint32) os.Error { + if length != 0 { + return FormatError("bad IEND length") } + return d.verifyChecksum() +} + +func (d *decoder) parseChunk() os.Error { + // Read the length and chunk type. + n, err := io.ReadFull(d.r, d.tmp[:8]) if err != nil { return err } - crc := crc32.NewIEEE() - crc.Write(d.tmp[0:4]) + length := binary.BigEndian.Uint32(d.tmp[:4]) + d.crc.Reset() + d.crc.Write(d.tmp[4:8]) // Read the chunk data. - switch string(d.tmp[0:4]) { + switch string(d.tmp[4:8]) { case "IHDR": if d.stage != dsStart { return chunkOrderError } d.stage = dsSeenIHDR - err = d.parseIHDR(r, crc, length) + return d.parseIHDR(length) case "PLTE": if d.stage != dsSeenIHDR { return chunkOrderError } d.stage = dsSeenPLTE - err = d.parsePLTE(r, crc, length) + return d.parsePLTE(length) case "tRNS": if d.stage != dsSeenPLTE { return chunkOrderError } - err = d.parsetRNS(r, crc, length) + return d.parsetRNS(length) case "IDAT": if d.stage < dsSeenIHDR || d.stage > dsSeenIDAT || (d.cb == cbP8 && d.stage == dsSeenIHDR) { return chunkOrderError } d.stage = dsSeenIDAT - err = d.parseIDAT(r, crc, length) + return d.parseIDAT(length) case "IEND": if d.stage != dsSeenIDAT { return chunkOrderError } d.stage = dsSeenIEND - err = d.parseIEND(r, crc, length) - default: - // Ignore this chunk (of a known length). - var ignored [4096]byte - for length > 0 { - n, err = io.ReadFull(r, ignored[0:min(len(ignored), int(length))]) - if err != nil { - return err - } - crc.Write(ignored[0:n]) - length -= uint32(n) - } + return d.parseIEND(length) } - if err != nil { - return err + // Ignore this chunk (of a known length). + var ignored [4096]byte + for length > 0 { + n, err = io.ReadFull(d.r, ignored[:min(len(ignored), int(length))]) + if err != nil { + return err + } + d.crc.Write(ignored[:n]) + length -= uint32(n) } + return d.verifyChecksum() +} - // Read the checksum. - n, err = io.ReadFull(r, d.tmp[0:4]) - if err == os.EOF { - return io.ErrUnexpectedEOF - } - if err != nil { +func (d *decoder) verifyChecksum() os.Error { + if _, err := io.ReadFull(d.r, d.tmp[:4]); err != nil { return err } - if parseUint32(d.tmp[0:4]) != crc.Sum32() { + if binary.BigEndian.Uint32(d.tmp[:4]) != d.crc.Sum32() { return FormatError("invalid checksum") } return nil } -func (d *decoder) checkHeader(r io.Reader) os.Error { - _, err := io.ReadFull(r, d.tmp[0:8]) +func (d *decoder) checkHeader() os.Error { + _, err := io.ReadFull(d.r, d.tmp[:len(pngHeader)]) if err != nil { return err } - if string(d.tmp[0:8]) != pngHeader { + if string(d.tmp[:len(pngHeader)]) != pngHeader { return FormatError("not a PNG file") } return nil @@ -634,42 +622,45 @@ func (d *decoder) checkHeader(r io.Reader) os.Error { // Decode reads a PNG image from r and returns it as an image.Image. // The type of Image returned depends on the PNG contents. func Decode(r io.Reader) (image.Image, os.Error) { - var d decoder - err := d.checkHeader(r) - if err != nil { - return nil, err + d := &decoder{ + r: r, + crc: crc32.NewIEEE(), } - for d.stage != dsSeenIEND { - err = d.parseChunk(r) - if err != nil { - break + if err := d.checkHeader(); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF } + return nil, err } - var img image.Image - if d.idatWriter != nil { - d.idatWriter.Close() - ie := <-d.idatDone - if err == nil { - img, err = ie.img, ie.err + for d.stage != dsSeenIEND { + if err := d.parseChunk(); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err } } - if err != nil { - return nil, err - } - return img, nil + return d.img, nil } // DecodeConfig returns the color model and dimensions of a PNG image without // decoding the entire image. func DecodeConfig(r io.Reader) (image.Config, os.Error) { - var d decoder - err := d.checkHeader(r) - if err != nil { + d := &decoder{ + r: r, + crc: crc32.NewIEEE(), + } + if err := d.checkHeader(); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } return image.Config{}, err } for { - err = d.parseChunk(r) - if err != nil { + if err := d.parseChunk(); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } return image.Config{}, err } if d.stage == dsSeenIHDR && d.cb != cbP8 { @@ -679,26 +670,26 @@ func DecodeConfig(r io.Reader) (image.Config, os.Error) { break } } - var cm image.ColorModel + var cm color.Model switch d.cb { case cbG1, cbG2, cbG4, cbG8: - cm = image.GrayColorModel + cm = color.GrayModel case cbGA8: - cm = image.NRGBAColorModel + cm = color.NRGBAModel case cbTC8: - cm = image.RGBAColorModel + cm = color.RGBAModel case cbP1, cbP2, cbP4, cbP8: cm = d.palette case cbTCA8: - cm = image.NRGBAColorModel + cm = color.NRGBAModel case cbG16: - cm = image.Gray16ColorModel + cm = color.Gray16Model case cbGA16: - cm = image.NRGBA64ColorModel + cm = color.NRGBA64Model case cbTC16: - cm = image.RGBA64ColorModel + cm = color.RGBA64Model case cbTCA16: - cm = image.NRGBA64ColorModel + cm = color.NRGBA64Model } return image.Config{cm, d.width, d.height}, nil } diff --git a/libgo/go/image/png/reader_test.go b/libgo/go/image/png/reader_test.go index bcc1a3d..48d0613 100644 --- a/libgo/go/image/png/reader_test.go +++ b/libgo/go/image/png/reader_test.go @@ -8,8 +8,10 @@ import ( "bufio" "fmt" "image" + "image/color" "io" "os" + "strings" "testing" ) @@ -41,7 +43,7 @@ var filenamesShort = []string{ "basn6a16", } -func readPng(filename string) (image.Image, os.Error) { +func readPNG(filename string) (image.Image, os.Error) { f, err := os.Open(filename) if err != nil { return nil, err @@ -57,12 +59,12 @@ func sng(w io.WriteCloser, filename string, png image.Image) { cm := png.ColorModel() var bitdepth int switch cm { - case image.RGBAColorModel, image.NRGBAColorModel, image.AlphaColorModel, image.GrayColorModel: + case color.RGBAModel, color.NRGBAModel, color.AlphaModel, color.GrayModel: bitdepth = 8 default: bitdepth = 16 } - cpm, _ := cm.(image.PalettedColorModel) + cpm, _ := cm.(color.Palette) var paletted *image.Paletted if cpm != nil { switch { @@ -82,11 +84,11 @@ func sng(w io.WriteCloser, filename string, png image.Image) { io.WriteString(w, "#SNG: from "+filename+".png\nIHDR {\n") fmt.Fprintf(w, " width: %d; height: %d; bitdepth: %d;\n", bounds.Dx(), bounds.Dy(), bitdepth) switch { - case cm == image.RGBAColorModel, cm == image.RGBA64ColorModel: + case cm == color.RGBAModel, cm == color.RGBA64Model: io.WriteString(w, " using color;\n") - case cm == image.NRGBAColorModel, cm == image.NRGBA64ColorModel: + case cm == color.NRGBAModel, cm == color.NRGBA64Model: io.WriteString(w, " using color alpha;\n") - case cm == image.GrayColorModel, cm == image.Gray16ColorModel: + case cm == color.GrayModel, cm == color.Gray16Model: io.WriteString(w, " using grayscale;\n") case cpm != nil: io.WriteString(w, " using color palette;\n") @@ -129,34 +131,34 @@ func sng(w io.WriteCloser, filename string, png image.Image) { io.WriteString(w, "IMAGE {\n pixels hex\n") for y := bounds.Min.Y; y < bounds.Max.Y; y++ { switch { - case cm == image.GrayColorModel: + case cm == color.GrayModel: for x := bounds.Min.X; x < bounds.Max.X; x++ { - gray := png.At(x, y).(image.GrayColor) + gray := png.At(x, y).(color.Gray) fmt.Fprintf(w, "%02x", gray.Y) } - case cm == image.Gray16ColorModel: + case cm == color.Gray16Model: for x := bounds.Min.X; x < bounds.Max.X; x++ { - gray16 := png.At(x, y).(image.Gray16Color) + gray16 := png.At(x, y).(color.Gray16) fmt.Fprintf(w, "%04x ", gray16.Y) } - case cm == image.RGBAColorModel: + case cm == color.RGBAModel: for x := bounds.Min.X; x < bounds.Max.X; x++ { - rgba := png.At(x, y).(image.RGBAColor) + rgba := png.At(x, y).(color.RGBA) fmt.Fprintf(w, "%02x%02x%02x ", rgba.R, rgba.G, rgba.B) } - case cm == image.RGBA64ColorModel: + case cm == color.RGBA64Model: for x := bounds.Min.X; x < bounds.Max.X; x++ { - rgba64 := png.At(x, y).(image.RGBA64Color) + rgba64 := png.At(x, y).(color.RGBA64) fmt.Fprintf(w, "%04x%04x%04x ", rgba64.R, rgba64.G, rgba64.B) } - case cm == image.NRGBAColorModel: + case cm == color.NRGBAModel: for x := bounds.Min.X; x < bounds.Max.X; x++ { - nrgba := png.At(x, y).(image.NRGBAColor) + nrgba := png.At(x, y).(color.NRGBA) fmt.Fprintf(w, "%02x%02x%02x%02x ", nrgba.R, nrgba.G, nrgba.B, nrgba.A) } - case cm == image.NRGBA64ColorModel: + case cm == color.NRGBA64Model: for x := bounds.Min.X; x < bounds.Max.X; x++ { - nrgba64 := png.At(x, y).(image.NRGBA64Color) + nrgba64 := png.At(x, y).(color.NRGBA64) fmt.Fprintf(w, "%04x%04x%04x%04x ", nrgba64.R, nrgba64.G, nrgba64.B, nrgba64.A) } case cpm != nil: @@ -183,7 +185,7 @@ func TestReader(t *testing.T) { } for _, fn := range names { // Read the .png file. - img, err := readPng("testdata/pngsuite/" + fn + ".png") + img, err := readPNG("testdata/pngsuite/" + fn + ".png") if err != nil { t.Error(fn, err) continue @@ -192,7 +194,7 @@ func TestReader(t *testing.T) { if fn == "basn4a16" { // basn4a16.sng is gray + alpha but sng() will produce true color + alpha // so we just check a single random pixel. - c := img.At(2, 1).(image.NRGBA64Color) + c := img.At(2, 1).(color.NRGBA64) if c.R != 0x11a7 || c.G != 0x11a7 || c.B != 0x11a7 || c.A != 0x1085 { t.Error(fn, fmt.Errorf("wrong pixel value at (2, 1): %x", c)) } @@ -239,3 +241,29 @@ func TestReader(t *testing.T) { } } } + +var readerErrors = []struct { + file string + err string +}{ + {"invalid-zlib.png", "zlib checksum error"}, + {"invalid-crc32.png", "invalid checksum"}, + {"invalid-noend.png", "unexpected EOF"}, + {"invalid-trunc.png", "unexpected EOF"}, +} + +func TestReaderError(t *testing.T) { + for _, tt := range readerErrors { + img, err := readPNG("testdata/" + tt.file) + if err == nil { + t.Errorf("decoding %s: missing error", tt.file) + continue + } + if !strings.Contains(err.String(), tt.err) { + t.Errorf("decoding %s: %s, want %s", tt.file, err, tt.err) + } + if img != nil { + t.Errorf("decoding %s: have image + error", tt.file) + } + } +} diff --git a/libgo/go/image/png/testdata/invalid-crc32.png b/libgo/go/image/png/testdata/invalid-crc32.png Binary files differnew file mode 100644 index 0000000..e5be408 --- /dev/null +++ b/libgo/go/image/png/testdata/invalid-crc32.png diff --git a/libgo/go/image/png/testdata/invalid-noend.png b/libgo/go/image/png/testdata/invalid-noend.png Binary files differnew file mode 100644 index 0000000..9137270 --- /dev/null +++ b/libgo/go/image/png/testdata/invalid-noend.png diff --git a/libgo/go/image/png/testdata/invalid-trunc.png b/libgo/go/image/png/testdata/invalid-trunc.png Binary files differnew file mode 100644 index 0000000..d0748cf --- /dev/null +++ b/libgo/go/image/png/testdata/invalid-trunc.png diff --git a/libgo/go/image/png/testdata/invalid-zlib.png b/libgo/go/image/png/testdata/invalid-zlib.png Binary files differnew file mode 100644 index 0000000..c6d051c --- /dev/null +++ b/libgo/go/image/png/testdata/invalid-zlib.png diff --git a/libgo/go/image/png/writer.go b/libgo/go/image/png/writer.go index 55ca97e..b6103c6 100644 --- a/libgo/go/image/png/writer.go +++ b/libgo/go/image/png/writer.go @@ -9,6 +9,7 @@ import ( "compress/zlib" "hash/crc32" "image" + "image/color" "io" "os" "strconv" @@ -125,7 +126,7 @@ func (e *encoder) writeIHDR() { e.writeChunk(e.tmp[0:13], "IHDR") } -func (e *encoder) writePLTE(p image.PalettedColorModel) { +func (e *encoder) writePLTE(p color.Palette) { if len(p) < 1 || len(p) > 256 { e.err = FormatError("bad palette length: " + strconv.Itoa(len(p))) return @@ -139,7 +140,7 @@ func (e *encoder) writePLTE(p image.PalettedColorModel) { e.writeChunk(e.tmp[0:3*len(p)], "PLTE") } -func (e *encoder) maybeWritetRNS(p image.PalettedColorModel) { +func (e *encoder) maybeWritetRNS(p color.Palette) { last := -1 for i, c := range p { _, _, _, a := c.RGBA() @@ -160,10 +161,6 @@ func (e *encoder) maybeWritetRNS(p image.PalettedColorModel) { // // This method should only be called from writeIDATs (via writeImage). // No other code should treat an encoder as an io.Writer. -// -// Note that, because the zlib Reader may involve an io.Pipe, e.Write calls may -// occur on a separate go-routine than the e.writeIDATs call, and care should be -// taken that e's state (such as its tmp buffer) is not modified concurrently. func (e *encoder) Write(b []byte) (int, os.Error) { e.writeChunk(b, "IDAT") if e.err != nil { @@ -310,7 +307,7 @@ func writeImage(w io.Writer, m image.Image, cb int) os.Error { switch cb { case cbG8: for x := b.Min.X; x < b.Max.X; x++ { - c := image.GrayColorModel.Convert(m.At(x, y)).(image.GrayColor) + c := color.GrayModel.Convert(m.At(x, y)).(color.Gray) cr[0][i] = c.Y i++ } @@ -336,13 +333,20 @@ func writeImage(w io.Writer, m image.Image, cb int) os.Error { } } case cbP8: - paletted := m.(*image.Paletted) - offset := (y - b.Min.Y) * paletted.Stride - copy(cr[0][1:], paletted.Pix[offset:offset+b.Dx()]) + if p, _ := m.(*image.Paletted); p != nil { + offset := (y - b.Min.Y) * p.Stride + copy(cr[0][1:], p.Pix[offset:offset+b.Dx()]) + } else { + pi := m.(image.PalettedImage) + for x := b.Min.X; x < b.Max.X; x++ { + cr[0][i] = pi.ColorIndexAt(x, y) + i += 1 + } + } case cbTCA8: // Convert from image.Image (which is alpha-premultiplied) to PNG's non-alpha-premultiplied. for x := b.Min.X; x < b.Max.X; x++ { - c := image.NRGBAColorModel.Convert(m.At(x, y)).(image.NRGBAColor) + c := color.NRGBAModel.Convert(m.At(x, y)).(color.NRGBA) cr[0][i+0] = c.R cr[0][i+1] = c.G cr[0][i+2] = c.B @@ -351,7 +355,7 @@ func writeImage(w io.Writer, m image.Image, cb int) os.Error { } case cbG16: for x := b.Min.X; x < b.Max.X; x++ { - c := image.Gray16ColorModel.Convert(m.At(x, y)).(image.Gray16Color) + c := color.Gray16Model.Convert(m.At(x, y)).(color.Gray16) cr[0][i+0] = uint8(c.Y >> 8) cr[0][i+1] = uint8(c.Y) i += 2 @@ -371,7 +375,7 @@ func writeImage(w io.Writer, m image.Image, cb int) os.Error { case cbTCA16: // Convert from image.Image (which is alpha-premultiplied) to PNG's non-alpha-premultiplied. for x := b.Min.X; x < b.Max.X; x++ { - c := image.NRGBA64ColorModel.Convert(m.At(x, y)).(image.NRGBA64Color) + c := color.NRGBA64Model.Convert(m.At(x, y)).(color.NRGBA64) cr[0][i+0] = uint8(c.R >> 8) cr[0][i+1] = uint8(c.R) cr[0][i+2] = uint8(c.G >> 8) @@ -432,16 +436,21 @@ func Encode(w io.Writer, m image.Image) os.Error { var e encoder e.w = w e.m = m - pal, _ := m.(*image.Paletted) + + var pal color.Palette + // cbP8 encoding needs PalettedImage's ColorIndexAt method. + if _, ok := m.(image.PalettedImage); ok { + pal, _ = m.ColorModel().(color.Palette) + } if pal != nil { e.cb = cbP8 } else { switch m.ColorModel() { - case image.GrayColorModel: + case color.GrayModel: e.cb = cbG8 - case image.Gray16ColorModel: + case color.Gray16Model: e.cb = cbG16 - case image.RGBAColorModel, image.NRGBAColorModel, image.AlphaColorModel: + case color.RGBAModel, color.NRGBAModel, color.AlphaModel: if opaque(m) { e.cb = cbTC8 } else { @@ -459,8 +468,8 @@ func Encode(w io.Writer, m image.Image) os.Error { _, e.err = io.WriteString(w, pngHeader) e.writeIHDR() if pal != nil { - e.writePLTE(pal.Palette) - e.maybeWritetRNS(pal.Palette) + e.writePLTE(pal) + e.maybeWritetRNS(pal) } e.writeIDATs() e.writeIEND() diff --git a/libgo/go/image/png/writer_test.go b/libgo/go/image/png/writer_test.go index 1599791..e517173 100644 --- a/libgo/go/image/png/writer_test.go +++ b/libgo/go/image/png/writer_test.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "image" + "image/color" "io/ioutil" "os" "testing" @@ -56,13 +57,13 @@ func TestWriter(t *testing.T) { for _, fn := range names { qfn := "testdata/pngsuite/" + fn + ".png" // Read the image. - m0, err := readPng(qfn) + m0, err := readPNG(qfn) if err != nil { t.Error(fn, err) continue } // Read the image again, encode it, and decode it. - m1, err := readPng(qfn) + m1, err := readPNG(qfn) if err != nil { t.Error(fn, err) return @@ -82,10 +83,10 @@ func TestWriter(t *testing.T) { } func TestSubImage(t *testing.T) { - m0 := image.NewRGBA(256, 256) + m0 := image.NewRGBA(image.Rect(0, 0, 256, 256)) for y := 0; y < 256; y++ { for x := 0; x < 256; x++ { - m0.Set(x, y, image.RGBAColor{uint8(x), uint8(y), 0, 255}) + m0.Set(x, y, color.RGBA{uint8(x), uint8(y), 0, 255}) } } m0 = m0.SubImage(image.Rect(50, 30, 250, 130)).(*image.RGBA) @@ -103,11 +104,10 @@ func TestSubImage(t *testing.T) { func BenchmarkEncodePaletted(b *testing.B) { b.StopTimer() - img := image.NewPaletted(640, 480, - []image.Color{ - image.RGBAColor{0, 0, 0, 255}, - image.RGBAColor{255, 255, 255, 255}, - }) + img := image.NewPaletted(image.Rect(0, 0, 640, 480), color.Palette{ + color.RGBA{0, 0, 0, 255}, + color.RGBA{255, 255, 255, 255}, + }) b.SetBytes(640 * 480 * 1) b.StartTimer() for i := 0; i < b.N; i++ { @@ -117,12 +117,12 @@ func BenchmarkEncodePaletted(b *testing.B) { func BenchmarkEncodeRGBOpaque(b *testing.B) { b.StopTimer() - img := image.NewRGBA(640, 480) + img := image.NewRGBA(image.Rect(0, 0, 640, 480)) // Set all pixels to 0xFF alpha to force opaque mode. bo := img.Bounds() for y := bo.Min.Y; y < bo.Max.Y; y++ { for x := bo.Min.X; x < bo.Max.X; x++ { - img.Set(x, y, image.RGBAColor{0, 0, 0, 255}) + img.Set(x, y, color.RGBA{0, 0, 0, 255}) } } if !img.Opaque() { @@ -137,7 +137,7 @@ func BenchmarkEncodeRGBOpaque(b *testing.B) { func BenchmarkEncodeRGBA(b *testing.B) { b.StopTimer() - img := image.NewRGBA(640, 480) + img := image.NewRGBA(image.Rect(0, 0, 640, 480)) if img.Opaque() { panic("expected image to not be opaque") } diff --git a/libgo/go/image/tiff/compress.go b/libgo/go/image/tiff/compress.go new file mode 100644 index 0000000..e89aa6d --- /dev/null +++ b/libgo/go/image/tiff/compress.go @@ -0,0 +1,60 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tiff + +import ( + "bufio" + "io" + "os" +) + +type byteReader interface { + io.Reader + io.ByteReader +} + +// unpackBits decodes the PackBits-compressed data in src and returns the +// uncompressed data. +// +// The PackBits compression format is described in section 9 (p. 42) +// of the TIFF spec. +func unpackBits(r io.Reader) ([]byte, os.Error) { + buf := make([]byte, 128) + dst := make([]byte, 0, 1024) + br, ok := r.(byteReader) + if !ok { + br = bufio.NewReader(r) + } + + for { + b, err := br.ReadByte() + if err != nil { + if err == os.EOF { + return dst, nil + } + return nil, err + } + code := int(int8(b)) + switch { + case code >= 0: + n, err := io.ReadFull(br, buf[:code+1]) + if err != nil { + return nil, err + } + dst = append(dst, buf[:n]...) + case code == -128: + // No-op. + default: + if b, err = br.ReadByte(); err != nil { + return nil, err + } + for j := 0; j < 1-code; j++ { + buf[j] = b + } + dst = append(dst, buf[:1-code]...) + } + } + panic("unreachable") +} diff --git a/libgo/go/image/tiff/reader.go b/libgo/go/image/tiff/reader.go index f565266..c452f5d 100644 --- a/libgo/go/image/tiff/reader.go +++ b/libgo/go/image/tiff/reader.go @@ -12,6 +12,7 @@ import ( "compress/zlib" "encoding/binary" "image" + "image/color" "io" "io/ioutil" "os" @@ -45,7 +46,7 @@ type decoder struct { config image.Config mode imageMode features map[int][]uint - palette []image.Color + palette []color.Color buf []byte off int // Current offset in buf. @@ -129,9 +130,9 @@ func (d *decoder) parseIFD(p []byte) os.Error { if len(val)%3 != 0 || numcolors <= 0 || numcolors > 256 { return FormatError("bad ColorMap length") } - d.palette = make([]image.Color, numcolors) + d.palette = make([]color.Color, numcolors) for i := 0; i < numcolors; i++ { - d.palette[i] = image.RGBA64Color{ + d.palette[i] = color.RGBA64{ uint16(val[i]), uint16(val[i+numcolors]), uint16(val[i+2*numcolors]), @@ -180,22 +181,21 @@ func (d *decoder) flushBits() { // decode decodes the raw data of an image. // It reads from d.buf and writes the strip with ymin <= y < ymax into dst. func (d *decoder) decode(dst image.Image, ymin, ymax int) os.Error { - spp := len(d.features[tBitsPerSample]) // samples per pixel d.off = 0 - width := dst.Bounds().Dx() // Apply horizontal predictor if necessary. // In this case, p contains the color difference to the preceding pixel. // See page 64-65 of the spec. if d.firstVal(tPredictor) == prHorizontal && d.firstVal(tBitsPerSample) == 8 { + var off int + spp := len(d.features[tBitsPerSample]) // samples per pixel for y := ymin; y < ymax; y++ { - d.off += spp - for x := 0; x < (width-1)*spp; x++ { - d.buf[d.off] += d.buf[d.off-spp] - d.off++ + off += spp + for x := 0; x < (dst.Bounds().Dx()-1)*spp; x++ { + d.buf[off] += d.buf[off-spp] + off++ } } - d.off = 0 } switch d.mode { @@ -209,7 +209,7 @@ func (d *decoder) decode(dst image.Image, ymin, ymax int) os.Error { if d.mode == mGrayInvert { v = 0xff - v } - img.SetGray(x, y, image.GrayColor{v}) + img.SetGray(x, y, color.Gray{v}) } d.flushBits() } @@ -224,28 +224,32 @@ func (d *decoder) decode(dst image.Image, ymin, ymax int) os.Error { } case mRGB: img := dst.(*image.RGBA) - for y := ymin; y < ymax; y++ { - for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { - img.SetRGBA(x, y, image.RGBAColor{d.buf[d.off], d.buf[d.off+1], d.buf[d.off+2], 0xff}) - d.off += spp - } + min := (ymin-img.Rect.Min.Y)*img.Stride - img.Rect.Min.X*4 + max := (ymax-img.Rect.Min.Y)*img.Stride - img.Rect.Min.X*4 + var off int + for i := min; i < max; i += 4 { + img.Pix[i+0] = d.buf[off+0] + img.Pix[i+1] = d.buf[off+1] + img.Pix[i+2] = d.buf[off+2] + img.Pix[i+3] = 0xff + off += 3 } case mNRGBA: img := dst.(*image.NRGBA) - for y := ymin; y < ymax; y++ { - for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { - img.SetNRGBA(x, y, image.NRGBAColor{d.buf[d.off], d.buf[d.off+1], d.buf[d.off+2], d.buf[d.off+3]}) - d.off += spp - } + min := (ymin-img.Rect.Min.Y)*img.Stride - img.Rect.Min.X*4 + max := (ymax-img.Rect.Min.Y)*img.Stride - img.Rect.Min.X*4 + if len(d.buf) != max-min { + return FormatError("short data strip") } + copy(img.Pix[min:max], d.buf) case mRGBA: img := dst.(*image.RGBA) - for y := ymin; y < ymax; y++ { - for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { - img.SetRGBA(x, y, image.RGBAColor{d.buf[d.off], d.buf[d.off+1], d.buf[d.off+2], d.buf[d.off+3]}) - d.off += spp - } + min := (ymin-img.Rect.Min.Y)*img.Stride - img.Rect.Min.X*4 + max := (ymax-img.Rect.Min.Y)*img.Stride - img.Rect.Min.X*4 + if len(d.buf) != max-min { + return FormatError("short data strip") } + copy(img.Pix[min:max], d.buf) } return nil @@ -305,10 +309,13 @@ func newDecoder(r io.Reader) (*decoder, os.Error) { return nil, UnsupportedError("non-8-bit RGB image") } } - d.config.ColorModel = image.RGBAColorModel + d.config.ColorModel = color.RGBAModel // RGB images normally have 3 samples per pixel. // If there are more, ExtraSamples (p. 31-32 of the spec) // gives their meaning (usually an alpha channel). + // + // This implementation does not support extra samples + // of an unspecified type. switch len(d.features[tBitsPerSample]) { case 3: d.mode = mRGB @@ -318,23 +325,22 @@ func newDecoder(r io.Reader) (*decoder, os.Error) { d.mode = mRGBA case 2: d.mode = mNRGBA - d.config.ColorModel = image.NRGBAColorModel + d.config.ColorModel = color.NRGBAModel default: - // The extra sample is discarded. - d.mode = mRGB + return nil, FormatError("wrong number of samples for RGB") } default: return nil, FormatError("wrong number of samples for RGB") } case pPaletted: d.mode = mPaletted - d.config.ColorModel = image.PalettedColorModel(d.palette) + d.config.ColorModel = color.Palette(d.palette) case pWhiteIsZero: d.mode = mGrayInvert - d.config.ColorModel = image.GrayColorModel + d.config.ColorModel = color.GrayModel case pBlackIsZero: d.mode = mGray - d.config.ColorModel = image.GrayColorModel + d.config.ColorModel = color.GrayModel default: return nil, UnsupportedError("color model") } @@ -373,13 +379,13 @@ func Decode(r io.Reader) (img image.Image, err os.Error) { switch d.mode { case mGray, mGrayInvert: - img = image.NewGray(d.config.Width, d.config.Height) + img = image.NewGray(image.Rect(0, 0, d.config.Width, d.config.Height)) case mPaletted: - img = image.NewPaletted(d.config.Width, d.config.Height, d.palette) + img = image.NewPaletted(image.Rect(0, 0, d.config.Width, d.config.Height), d.palette) case mNRGBA: - img = image.NewNRGBA(d.config.Width, d.config.Height) + img = image.NewNRGBA(image.Rect(0, 0, d.config.Width, d.config.Height)) case mRGB, mRGBA: - img = image.NewRGBA(d.config.Width, d.config.Height) + img = image.NewRGBA(image.Rect(0, 0, d.config.Width, d.config.Height)) } for i := 0; i < numStrips; i++ { @@ -406,6 +412,8 @@ func Decode(r io.Reader) (img image.Image, err os.Error) { } d.buf, err = ioutil.ReadAll(r) r.Close() + case cPackBits: + d.buf, err = unpackBits(io.NewSectionReader(d.r, offset, n)) default: err = UnsupportedError("compression") } diff --git a/libgo/go/image/tiff/reader_test.go b/libgo/go/image/tiff/reader_test.go index f2122c4..86b7dc3 100644 --- a/libgo/go/image/tiff/reader_test.go +++ b/libgo/go/image/tiff/reader_test.go @@ -5,10 +5,18 @@ package tiff import ( + "image" + "io/ioutil" "os" + "strings" "testing" ) +// Read makes *buffer implements io.Reader, so that we can pass one to Decode. +func (*buffer) Read([]byte) (int, os.Error) { + panic("unimplemented") +} + // TestNoRPS tries to decode an image that has no RowsPerStrip tag. // The tag is mandatory according to the spec but some software omits // it in the case of a single strip. @@ -23,3 +31,89 @@ func TestNoRPS(t *testing.T) { t.Fatal(err) } } + +// TestUnpackBits tests the decoding of PackBits-encoded data. +func TestUnpackBits(t *testing.T) { + var unpackBitsTests = []struct { + compressed string + uncompressed string + }{{ + // Example data from Wikipedia. + "\xfe\xaa\x02\x80\x00\x2a\xfd\xaa\x03\x80\x00\x2a\x22\xf7\xaa", + "\xaa\xaa\xaa\x80\x00\x2a\xaa\xaa\xaa\xaa\x80\x00\x2a\x22\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + }} + for _, u := range unpackBitsTests { + buf, err := unpackBits(strings.NewReader(u.compressed)) + if err != nil { + t.Fatal(err) + } + if string(buf) != u.uncompressed { + t.Fatalf("unpackBits: want %x, got %x", u.uncompressed, buf) + } + } +} + +// TestDecompress tests that decoding some TIFF images that use different +// compression formats result in the same pixel data. +func TestDecompress(t *testing.T) { + var decompressTests = []string{ + "bw-uncompressed.tiff", + "bw-deflate.tiff", + "bw-packbits.tiff", + } + var img0 image.Image + for _, name := range decompressTests { + f, err := os.Open("testdata/" + name) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if img0 == nil { + img0, err = Decode(f) + if err != nil { + t.Fatalf("decoding %s: %v", name, err) + } + continue + } + + img1, err := Decode(f) + if err != nil { + t.Fatalf("decoding %s: %v", name, err) + } + b := img1.Bounds() + // Compare images. + if !b.Eq(img0.Bounds()) { + t.Fatalf("wrong image size: want %s, got %s", img0.Bounds(), b) + } + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + c0 := img0.At(x, y) + c1 := img1.At(x, y) + r0, g0, b0, a0 := c0.RGBA() + r1, g1, b1, a1 := c1.RGBA() + if r0 != r1 || g0 != g1 || b0 != b1 || a0 != a1 { + t.Fatalf("pixel at (%d, %d) has wrong color: want %v, got %v", x, y, c0, c1) + } + } + } + } +} + +const filename = "testdata/video-001-uncompressed.tiff" + +// BenchmarkDecode benchmarks the decoding of an image. +func BenchmarkDecode(b *testing.B) { + b.StopTimer() + contents, err := ioutil.ReadFile(filename) + if err != nil { + panic(err) + } + r := &buffer{buf: contents} + b.StartTimer() + for i := 0; i < b.N; i++ { + _, err := Decode(r) + if err != nil { + panic(err) + } + } +} diff --git a/libgo/go/image/tiff/testdata/bw-deflate.tiff b/libgo/go/image/tiff/testdata/bw-deflate.tiff Binary files differnew file mode 100644 index 0000000..137a0c3 --- /dev/null +++ b/libgo/go/image/tiff/testdata/bw-deflate.tiff diff --git a/libgo/go/image/tiff/testdata/bw-packbits.tiff b/libgo/go/image/tiff/testdata/bw-packbits.tiff Binary files differnew file mode 100644 index 0000000..d59fa4a --- /dev/null +++ b/libgo/go/image/tiff/testdata/bw-packbits.tiff diff --git a/libgo/go/image/tiff/testdata/bw-uncompressed.tiff b/libgo/go/image/tiff/testdata/bw-uncompressed.tiff Binary files differnew file mode 100644 index 0000000..8390f11 --- /dev/null +++ b/libgo/go/image/tiff/testdata/bw-uncompressed.tiff diff --git a/libgo/go/image/tiff/testdata/video-001-uncompressed.tiff b/libgo/go/image/tiff/testdata/video-001-uncompressed.tiff Binary files differnew file mode 100644 index 0000000..fad1471 --- /dev/null +++ b/libgo/go/image/tiff/testdata/video-001-uncompressed.tiff diff --git a/libgo/go/image/ycbcr/ycbcr.go b/libgo/go/image/ycbcr/ycbcr.go index f2de3d6..84a35a3 100644 --- a/libgo/go/image/ycbcr/ycbcr.go +++ b/libgo/go/image/ycbcr/ycbcr.go @@ -15,6 +15,7 @@ package ycbcr import ( "image" + "image/color" ) // RGBToYCbCr converts an RGB triple to a YCbCr triple. All components lie @@ -92,7 +93,7 @@ func (c YCbCrColor) RGBA() (uint32, uint32, uint32, uint32) { return uint32(r) * 0x101, uint32(g) * 0x101, uint32(b) * 0x101, 0xffff } -func toYCbCrColor(c image.Color) image.Color { +func toYCbCrColor(c color.Color) color.Color { if _, ok := c.(YCbCrColor); ok { return c } @@ -102,7 +103,7 @@ func toYCbCrColor(c image.Color) image.Color { } // YCbCrColorModel is the color model for YCbCrColor. -var YCbCrColorModel image.ColorModel = image.ColorModelFunc(toYCbCrColor) +var YCbCrColorModel color.Model = color.ModelFunc(toYCbCrColor) // SubsampleRatio is the chroma subsample ratio used in a YCbCr image. type SubsampleRatio int @@ -133,7 +134,7 @@ type YCbCr struct { Rect image.Rectangle } -func (p *YCbCr) ColorModel() image.ColorModel { +func (p *YCbCr) ColorModel() color.Model { return YCbCrColorModel } @@ -141,7 +142,7 @@ func (p *YCbCr) Bounds() image.Rectangle { return p.Rect } -func (p *YCbCr) At(x, y int) image.Color { +func (p *YCbCr) At(x, y int) color.Color { if !(image.Point{x, y}.In(p.Rect)) { return YCbCrColor{} } diff --git a/libgo/go/index/suffixarray/qsufsort.go b/libgo/go/index/suffixarray/qsufsort.go index 30c1104..c69be43 100644 --- a/libgo/go/index/suffixarray/qsufsort.go +++ b/libgo/go/index/suffixarray/qsufsort.go @@ -37,7 +37,7 @@ func qsufsort(data []byte) []int { inv := initGroups(sa, data) // the index starts 1-ordered - sufSortable := &suffixSortable{sa, inv, 1} + sufSortable := &suffixSortable{sa: sa, inv: inv, h: 1} for sa[0] > -len(sa) { // until all suffixes are one big sorted group // The suffixes are h-ordered, make them 2*h-ordered @@ -135,6 +135,7 @@ type suffixSortable struct { sa []int inv []int h int + buf []int // common scratch space } func (x *suffixSortable) Len() int { return len(x.sa) } @@ -142,7 +143,7 @@ func (x *suffixSortable) Less(i, j int) bool { return x.inv[x.sa[i]+x.h] < x.inv func (x *suffixSortable) Swap(i, j int) { x.sa[i], x.sa[j] = x.sa[j], x.sa[i] } func (x *suffixSortable) updateGroups(offset int) { - bounds := make([]int, 0, 4) + bounds := x.buf[0:0] group := x.inv[x.sa[0]+x.h] for i := 1; i < len(x.sa); i++ { if g := x.inv[x.sa[i]+x.h]; g > group { @@ -151,6 +152,7 @@ func (x *suffixSortable) updateGroups(offset int) { } } bounds = append(bounds, len(x.sa)) + x.buf = bounds // update the group numberings after all new groups are determined prev := 0 diff --git a/libgo/go/index/suffixarray/suffixarray.go b/libgo/go/index/suffixarray/suffixarray.go index 82e98d2..174460c 100644 --- a/libgo/go/index/suffixarray/suffixarray.go +++ b/libgo/go/index/suffixarray/suffixarray.go @@ -18,6 +18,9 @@ package suffixarray import ( "bytes" + "encoding/binary" + "io" + "os" "regexp" "sort" ) @@ -25,7 +28,7 @@ import ( // Index implements a suffix array for fast substring search. type Index struct { data []byte - sa []int // suffix array for data + sa []int // suffix array for data; len(sa) == len(data) } // New creates a new Index for data. @@ -34,6 +37,129 @@ func New(data []byte) *Index { return &Index{data, qsufsort(data)} } +// writeInt writes an int x to w using buf to buffer the write. +func writeInt(w io.Writer, buf []byte, x int) os.Error { + binary.PutVarint(buf, int64(x)) + _, err := w.Write(buf[0:binary.MaxVarintLen64]) + return err +} + +// readInt reads an int x from r using buf to buffer the read and returns x. +func readInt(r io.Reader, buf []byte) (int, os.Error) { + _, err := io.ReadFull(r, buf[0:binary.MaxVarintLen64]) // ok to continue with error + x, _ := binary.Varint(buf) + return int(x), err +} + +// writeSlice writes data[:n] to w and returns n. +// It uses buf to buffer the write. +func writeSlice(w io.Writer, buf []byte, data []int) (n int, err os.Error) { + // encode as many elements as fit into buf + p := binary.MaxVarintLen64 + for ; n < len(data) && p+binary.MaxVarintLen64 <= len(buf); n++ { + p += binary.PutUvarint(buf[p:], uint64(data[n])) + } + + // update buffer size + binary.PutVarint(buf, int64(p)) + + // write buffer + _, err = w.Write(buf[0:p]) + return +} + +// readSlice reads data[:n] from r and returns n. +// It uses buf to buffer the read. +func readSlice(r io.Reader, buf []byte, data []int) (n int, err os.Error) { + // read buffer size + var size int + size, err = readInt(r, buf) + if err != nil { + return + } + + // read buffer w/o the size + if _, err = io.ReadFull(r, buf[binary.MaxVarintLen64:size]); err != nil { + return + } + + // decode as many elements as present in buf + for p := binary.MaxVarintLen64; p < size; n++ { + x, w := binary.Uvarint(buf[p:]) + data[n] = int(x) + p += w + } + + return +} + +const bufSize = 16 << 10 // reasonable for BenchmarkSaveRestore + +// Read reads the index from r into x; x must not be nil. +func (x *Index) Read(r io.Reader) os.Error { + // buffer for all reads + buf := make([]byte, bufSize) + + // read length + n, err := readInt(r, buf) + if err != nil { + return err + } + + // allocate space + if 2*n < cap(x.data) || cap(x.data) < n { + // new data is significantly smaller or larger then + // existing buffers - allocate new ones + x.data = make([]byte, n) + x.sa = make([]int, n) + } else { + // re-use existing buffers + x.data = x.data[0:n] + x.sa = x.sa[0:n] + } + + // read data + if _, err := io.ReadFull(r, x.data); err != nil { + return err + } + + // read index + for sa := x.sa; len(sa) > 0; { + n, err := readSlice(r, buf, sa) + if err != nil { + return err + } + sa = sa[n:] + } + return nil +} + +// Write writes the index x to w. +func (x *Index) Write(w io.Writer) os.Error { + // buffer for all writes + buf := make([]byte, bufSize) + + // write length + if err := writeInt(w, buf, len(x.data)); err != nil { + return err + } + + // write data + if _, err := w.Write(x.data); err != nil { + return err + } + + // write index + for sa := x.sa; len(sa) > 0; { + n, err := writeSlice(w, buf, sa) + if err != nil { + return err + } + sa = sa[n:] + } + return nil +} + // Bytes returns the data over which the index was created. // It must not be modified. // @@ -65,9 +191,10 @@ func (x *Index) lookupAll(s []byte) []int { func (x *Index) Lookup(s []byte, n int) (result []int) { if len(s) > 0 && n != 0 { matches := x.lookupAll(s) - if len(matches) < n || n < 0 { + if n < 0 || len(matches) < n { n = len(matches) } + // 0 <= n <= len(matches) if n > 0 { result = make([]int, n) copy(result, matches) diff --git a/libgo/go/index/suffixarray/suffixarray_test.go b/libgo/go/index/suffixarray/suffixarray_test.go index 0237485..f6b2f00 100644 --- a/libgo/go/index/suffixarray/suffixarray_test.go +++ b/libgo/go/index/suffixarray/suffixarray_test.go @@ -6,6 +6,7 @@ package suffixarray import ( "bytes" + "rand" "regexp" "sort" "strings" @@ -213,14 +214,44 @@ func (a *index) at(i int) []byte { return a.data[a.sa[i]:] } func testConstruction(t *testing.T, tc *testCase, x *Index) { if !sort.IsSorted((*index)(x)) { - t.Errorf("testConstruction failed %s", tc.name) + t.Errorf("failed testConstruction %s", tc.name) } } +func equal(x, y *Index) bool { + if !bytes.Equal(x.data, y.data) { + return false + } + for i, j := range x.sa { + if j != y.sa[i] { + return false + } + } + return true +} + +// returns the serialized index size +func testSaveRestore(t *testing.T, tc *testCase, x *Index) int { + var buf bytes.Buffer + if err := x.Write(&buf); err != nil { + t.Errorf("failed writing index %s (%s)", tc.name, err) + } + size := buf.Len() + var y Index + if err := y.Read(&buf); err != nil { + t.Errorf("failed reading index %s (%s)", tc.name, err) + } + if !equal(x, &y) { + t.Errorf("restored index doesn't match saved index %s", tc.name) + } + return size +} + func TestIndex(t *testing.T) { for _, tc := range testCases { x := New([]byte(tc.source)) testConstruction(t, &tc, x) + testSaveRestore(t, &tc, x) testLookups(t, &tc, x, 0) testLookups(t, &tc, x, 1) testLookups(t, &tc, x, 10) @@ -228,3 +259,46 @@ func TestIndex(t *testing.T) { testLookups(t, &tc, x, -1) } } + +// Of all possible inputs, the random bytes have the least amount of substring +// repetition, and the repeated bytes have the most. For most algorithms, +// the running time of every input will be between these two. +func benchmarkNew(b *testing.B, random bool) { + b.StopTimer() + data := make([]byte, 1e6) + if random { + for i := range data { + data[i] = byte(rand.Intn(256)) + } + } + b.StartTimer() + for i := 0; i < b.N; i++ { + New(data) + } +} + +func BenchmarkNewIndexRandom(b *testing.B) { + benchmarkNew(b, true) +} +func BenchmarkNewIndexRepeat(b *testing.B) { + benchmarkNew(b, false) +} + +func BenchmarkSaveRestore(b *testing.B) { + b.StopTimer() + r := rand.New(rand.NewSource(0x5a77a1)) // guarantee always same sequence + data := make([]byte, 10<<20) // 10MB of data to index + for i := range data { + data[i] = byte(r.Intn(256)) + } + x := New(data) + size := testSaveRestore(nil, nil, x) // verify correctness + buf := bytes.NewBuffer(make([]byte, size)) // avoid growing + b.SetBytes(int64(size)) + b.StartTimer() + for i := 0; i < b.N; i++ { + x.Write(buf) + var y Index + y.Read(buf) + } +} diff --git a/libgo/go/io/io.go b/libgo/go/io/io.go index b879fe5..5520634 100644 --- a/libgo/go/io/io.go +++ b/libgo/go/io/io.go @@ -256,15 +256,15 @@ func ReadFull(r Reader, buf []byte) (n int, err os.Error) { return ReadAtLeast(r, buf, len(buf)) } -// Copyn copies n bytes (or until an error) from src to dst. +// CopyN copies n bytes (or until an error) from src to dst. // It returns the number of bytes copied and the earliest // error encountered while copying. Because Read can // return the full amount requested as well as an error -// (including os.EOF), so can Copyn. +// (including os.EOF), so can CopyN. // // If dst implements the ReaderFrom interface, // the copy is implemented by calling dst.ReadFrom(src). -func Copyn(dst Writer, src Reader, n int64) (written int64, err os.Error) { +func CopyN(dst Writer, src Reader, n int64) (written int64, err os.Error) { // If the writer has a ReadFrom method, use it to do the copy. // Avoids a buffer allocation and a copy. if rt, ok := dst.(ReaderFrom); ok { @@ -437,3 +437,27 @@ func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err os.Error) { // Size returns the size of the section in bytes. func (s *SectionReader) Size() int64 { return s.limit - s.base } + +// TeeReader returns a Reader that writes to w what it reads from r. +// All reads from r performed through it are matched with +// corresponding writes to w. There is no internal buffering - +// the write must complete before the read completes. +// Any error encountered while writing is reported as a read error. +func TeeReader(r Reader, w Writer) Reader { + return &teeReader{r, w} +} + +type teeReader struct { + r Reader + w Writer +} + +func (t *teeReader) Read(p []byte) (n int, err os.Error) { + n, err = t.r.Read(p) + if n > 0 { + if n, err := t.w.Write(p[:n]); err != nil { + return n, err + } + } + return +} diff --git a/libgo/go/io/io_test.go b/libgo/go/io/io_test.go index bc4f354..f1b23e9 100644 --- a/libgo/go/io/io_test.go +++ b/libgo/go/io/io_test.go @@ -19,7 +19,7 @@ type Buffer struct { WriterTo // conflicts with and hides bytes.Buffer's WriterTo. } -// Simple tests, primarily to verify the ReadFrom and WriteTo callouts inside Copy and Copyn. +// Simple tests, primarily to verify the ReadFrom and WriteTo callouts inside Copy and CopyN. func TestCopy(t *testing.T) { rb := new(Buffer) @@ -51,33 +51,33 @@ func TestCopyWriteTo(t *testing.T) { } } -func TestCopyn(t *testing.T) { +func TestCopyN(t *testing.T) { rb := new(Buffer) wb := new(Buffer) rb.WriteString("hello, world.") - Copyn(wb, rb, 5) + CopyN(wb, rb, 5) if wb.String() != "hello" { - t.Errorf("Copyn did not work properly") + t.Errorf("CopyN did not work properly") } } -func TestCopynReadFrom(t *testing.T) { +func TestCopyNReadFrom(t *testing.T) { rb := new(Buffer) wb := new(bytes.Buffer) // implements ReadFrom. rb.WriteString("hello") - Copyn(wb, rb, 5) + CopyN(wb, rb, 5) if wb.String() != "hello" { - t.Errorf("Copyn did not work properly") + t.Errorf("CopyN did not work properly") } } -func TestCopynWriteTo(t *testing.T) { +func TestCopyNWriteTo(t *testing.T) { rb := new(bytes.Buffer) // implements WriteTo. wb := new(Buffer) rb.WriteString("hello, world.") - Copyn(wb, rb, 5) + CopyN(wb, rb, 5) if wb.String() != "hello" { - t.Errorf("Copyn did not work properly") + t.Errorf("CopyN did not work properly") } } @@ -89,30 +89,30 @@ func (w *noReadFrom) Write(p []byte) (n int, err os.Error) { return w.w.Write(p) } -func TestCopynEOF(t *testing.T) { +func TestCopyNEOF(t *testing.T) { // Test that EOF behavior is the same regardless of whether - // argument to Copyn has ReadFrom. + // argument to CopyN has ReadFrom. b := new(bytes.Buffer) - n, err := Copyn(&noReadFrom{b}, strings.NewReader("foo"), 3) + n, err := CopyN(&noReadFrom{b}, strings.NewReader("foo"), 3) if n != 3 || err != nil { - t.Errorf("Copyn(noReadFrom, foo, 3) = %d, %v; want 3, nil", n, err) + t.Errorf("CopyN(noReadFrom, foo, 3) = %d, %v; want 3, nil", n, err) } - n, err = Copyn(&noReadFrom{b}, strings.NewReader("foo"), 4) + n, err = CopyN(&noReadFrom{b}, strings.NewReader("foo"), 4) if n != 3 || err != os.EOF { - t.Errorf("Copyn(noReadFrom, foo, 4) = %d, %v; want 3, EOF", n, err) + t.Errorf("CopyN(noReadFrom, foo, 4) = %d, %v; want 3, EOF", n, err) } - n, err = Copyn(b, strings.NewReader("foo"), 3) // b has read from + n, err = CopyN(b, strings.NewReader("foo"), 3) // b has read from if n != 3 || err != nil { - t.Errorf("Copyn(bytes.Buffer, foo, 3) = %d, %v; want 3, nil", n, err) + t.Errorf("CopyN(bytes.Buffer, foo, 3) = %d, %v; want 3, nil", n, err) } - n, err = Copyn(b, strings.NewReader("foo"), 4) // b has read from + n, err = CopyN(b, strings.NewReader("foo"), 4) // b has read from if n != 3 || err != os.EOF { - t.Errorf("Copyn(bytes.Buffer, foo, 4) = %d, %v; want 3, EOF", n, err) + t.Errorf("CopyN(bytes.Buffer, foo, 4) = %d, %v; want 3, EOF", n, err) } } @@ -177,3 +177,30 @@ func testReadAtLeast(t *testing.T, rb ReadWriter) { t.Errorf("expected to have read 1 bytes, got %v", n) } } + +func TestTeeReader(t *testing.T) { + src := []byte("hello, world") + dst := make([]byte, len(src)) + rb := bytes.NewBuffer(src) + wb := new(bytes.Buffer) + r := TeeReader(rb, wb) + if n, err := ReadFull(r, dst); err != nil || n != len(src) { + t.Fatalf("ReadFull(r, dst) = %d, %v; want %d, nil", n, err, len(src)) + } + if !bytes.Equal(dst, src) { + t.Errorf("bytes read = %q want %q", dst, src) + } + if !bytes.Equal(wb.Bytes(), src) { + t.Errorf("bytes written = %q want %q", wb.Bytes(), src) + } + if n, err := r.Read(dst); n != 0 || err != os.EOF { + t.Errorf("r.Read at EOF = %d, %v want 0, EOF", n, err) + } + rb = bytes.NewBuffer(src) + pr, pw := Pipe() + pr.Close() + r = TeeReader(rb, pw) + if n, err := ReadFull(r, dst); n != 0 || err != os.EPIPE { + t.Errorf("closed tee: ReadFull(r, dst) = %d, %v; want 0, EPIPE", n, err) + } +} diff --git a/libgo/go/io/ioutil/ioutil.go b/libgo/go/io/ioutil/ioutil.go index fffa132..dd50d96 100644 --- a/libgo/go/io/ioutil/ioutil.go +++ b/libgo/go/io/ioutil/ioutil.go @@ -104,6 +104,10 @@ func NopCloser(r io.Reader) io.ReadCloser { type devNull int +// devNull implements ReaderFrom as an optimization so io.Copy to +// ioutil.Discard can avoid doing unnecessary work. +var _ io.ReaderFrom = devNull(0) + func (devNull) Write(p []byte) (int, os.Error) { return len(p), nil } diff --git a/libgo/go/json/decode.go b/libgo/go/json/decode.go index b7129f9..31b15a4 100644 --- a/libgo/go/json/decode.go +++ b/libgo/go/json/decode.go @@ -22,17 +22,20 @@ import ( // Unmarshal parses the JSON-encoded data and stores the result // in the value pointed to by v. // -// Unmarshal traverses the value v recursively. -// If an encountered value implements the Unmarshaler interface, -// Unmarshal calls its UnmarshalJSON method with a well-formed -// JSON encoding. -// -// Otherwise, Unmarshal uses the inverse of the encodings that +// Unmarshal uses the inverse of the encodings that // Marshal uses, allocating maps, slices, and pointers as necessary, // with the following additional rules: // -// To unmarshal a JSON value into a nil interface value, the -// type stored in the interface value is one of: +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into an interface value, Unmarshal unmarshals +// the JSON into the concrete value contained in the interface value. +// If the interface value is nil, that is, has no concrete value stored in it, +// Unmarshal stores one of these in the interface value: // // bool, for JSON booleans // float64, for JSON numbers @@ -250,8 +253,8 @@ func (d *decodeState) value(v reflect.Value) { // indirect walks down v allocating pointers as needed, // until it gets to a non-pointer. // if it encounters an Unmarshaler, indirect stops and returns that. -// if wantptr is true, indirect stops at the last pointer. -func (d *decodeState) indirect(v reflect.Value, wantptr bool) (Unmarshaler, reflect.Value) { +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, reflect.Value) { // If v is a named type and is addressable, // start with its address, so that if the type has pointer methods, // we find them. @@ -277,7 +280,7 @@ func (d *decodeState) indirect(v reflect.Value, wantptr bool) (Unmarshaler, refl break } - if pv.Elem().Kind() != reflect.Ptr && wantptr && pv.CanSet() && !isUnmarshaler { + if pv.Elem().Kind() != reflect.Ptr && decodingNull && pv.CanSet() { return nil, pv } if pv.IsNil() { @@ -391,11 +394,6 @@ func (d *decodeState) array(v reflect.Value) { } } -// matchName returns true if key should be written to a field named name. -func matchName(key, name string) bool { - return strings.ToLower(key) == strings.ToLower(name) -} - // object consumes an object from d.data[d.off-1:], decoding into the value v. // the first byte of the object ('{') has been read already. func (d *decodeState) object(v reflect.Value) { @@ -485,24 +483,31 @@ func (d *decodeState) object(v reflect.Value) { var f reflect.StructField var ok bool st := sv.Type() - // First try for field with that tag. - if isValidTag(key) { - for i := 0; i < sv.NumField(); i++ { - f = st.Field(i) - tagName, _ := parseTag(f.Tag.Get("json")) - if tagName == key { - ok = true - break - } + for i := 0; i < sv.NumField(); i++ { + sf := st.Field(i) + tag := sf.Tag.Get("json") + if tag == "-" { + // Pretend this field doesn't exist. + continue + } + // First, tag match + tagName, _ := parseTag(tag) + if tagName == key { + f = sf + ok = true + break // no better match possible + } + // Second, exact field name match + if sf.Name == key { + f = sf + ok = true + } + // Third, case-insensitive field name match, + // but only if a better match hasn't already been seen + if !ok && strings.EqualFold(sf.Name, key) { + f = sf + ok = true } - } - if !ok { - // Second, exact match. - f, ok = st.FieldByName(key) - } - if !ok { - // Third, case-insensitive match. - f, ok = st.FieldByNameFunc(func(s string) bool { return matchName(key, s) }) } // Extract value; name must be exported. diff --git a/libgo/go/json/decode_test.go b/libgo/go/json/decode_test.go index 5f6c3f5..2c7cbc4 100644 --- a/libgo/go/json/decode_test.go +++ b/libgo/go/json/decode_test.go @@ -15,6 +15,7 @@ import ( type T struct { X string Y int + Z int `json:"-"` } type tx struct { @@ -42,7 +43,7 @@ var ( um0, um1 unmarshaler // target2 of unmarshaling ump = &um1 umtrue = unmarshaler{true} - umslice = []unmarshaler{unmarshaler{true}} + umslice = []unmarshaler{{true}} umslicep = new([]unmarshaler) umstruct = ustruct{unmarshaler{true}} ) @@ -68,6 +69,9 @@ var unmarshalTests = []unmarshalTest{ {`{"X": [1,2,3], "Y": 4}`, new(T), T{Y: 4}, &UnmarshalTypeError{"array", reflect.TypeOf("")}}, {`{"x": 1}`, new(tx), tx{}, &UnmarshalFieldError{"x", txType, txType.Field(0)}}, + // Z has a "-" tag. + {`{"Y": 1, "Z": 2}`, new(T), T{Y: 1}, nil}, + // syntax errors {`{"X": "foo", "Y"}`, nil, nil, &SyntaxError{"invalid character '}' after object key", 17}}, diff --git a/libgo/go/json/encode.go b/libgo/go/json/encode.go index 16be5e2..46abe436 100644 --- a/libgo/go/json/encode.go +++ b/libgo/go/json/encode.go @@ -24,8 +24,11 @@ import ( // Marshal returns the JSON encoding of v. // // Marshal traverses the value v recursively. -// If an encountered value implements the Marshaler interface, -// Marshal calls its MarshalJSON method to produce JSON. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. // // Otherwise, Marshal uses the following type-dependent default encodings: // @@ -40,18 +43,23 @@ import ( // []byte encodes as a base64-encoded string. // // Struct values encode as JSON objects. Each exported struct field -// becomes a member of the object unless the field is empty and its tag -// specifies the "omitempty" option. The empty values are false, 0, any +// becomes a member of the object unless +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option. +// The empty values are false, 0, any // nil pointer or interface value, and any array, slice, map, or string of // length zero. The object's default key string is the struct field name // but can be specified in the struct field's tag value. The "json" key in // struct field's tag value is the key name, followed by an optional comma // and options. Examples: // -// // Specifies that Field appears in JSON as key "myName" +// // Field is ignored by this package. +// Field int `json:"-"` +// +// // Field appears in JSON as key "myName". // Field int `json:"myName"` // -// // Specifies that Field appears in JSON as key "myName" and +// // Field appears in JSON as key "myName" and // // the field is omitted from the object if its value is empty, // // as defined above. // Field int `json:"myName,omitempty"` @@ -240,7 +248,7 @@ func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) { return } - if j, ok := v.Interface().(Marshaler); ok { + if j, ok := v.Interface().(Marshaler); ok && (v.Kind() != reflect.Ptr || !v.IsNil()) { b, err := j.MarshalJSON() if err == nil { // copy JSON into buffer, checking validity. @@ -298,6 +306,9 @@ func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) { } tag, omitEmpty, quoted := f.Name, false, false if tv := f.Tag.Get("json"); tv != "" { + if tv == "-" { + continue + } name, opts := parseTag(tv) if isValidTag(name) { tag = name diff --git a/libgo/go/json/encode_test.go b/libgo/go/json/encode_test.go index 012e9f1..f85bb62 100644 --- a/libgo/go/json/encode_test.go +++ b/libgo/go/json/encode_test.go @@ -13,6 +13,7 @@ import ( type Optionals struct { Sr string `json:"sr"` So string `json:"so,omitempty"` + Sw string `json:"-"` Ir int `json:"omitempty"` // actually named omitempty, not an option Io int `json:"io,omitempty"` @@ -33,6 +34,7 @@ var optionalsExpected = `{ func TestOmitEmpty(t *testing.T) { var o Optionals + o.Sw = "something" o.Mr = map[string]interface{}{} o.Mo = map[string]interface{}{} diff --git a/libgo/go/json/indent.go b/libgo/go/json/indent.go index 000da42..2a75303 100644 --- a/libgo/go/json/indent.go +++ b/libgo/go/json/indent.go @@ -59,6 +59,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) os.Error { needIndent := false depth := 0 for _, c := range src { + scan.bytes++ v := scan.step(&scan, int(c)) if v == scanSkipSpace { continue diff --git a/libgo/go/json/scanner_test.go b/libgo/go/json/scanner_test.go index 023e7c8..4d73eac 100644 --- a/libgo/go/json/scanner_test.go +++ b/libgo/go/json/scanner_test.go @@ -7,7 +7,9 @@ package json import ( "bytes" "math" + "os" "rand" + "reflect" "testing" ) @@ -136,6 +138,29 @@ func TestIndentBig(t *testing.T) { } } +type indentErrorTest struct { + in string + err os.Error +} + +var indentErrorTests = []indentErrorTest{ + {`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}}, + {`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}}, +} + +func TestIdentErrors(t *testing.T) { + for i, tt := range indentErrorTests { + slice := make([]uint8, 0) + buf := bytes.NewBuffer(slice) + if err := Indent(buf, []uint8(tt.in), "", ""); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("#%d: Indent: %#v", i, err) + continue + } + } + } +} + func TestNextValueBig(t *testing.T) { initBig() var scan scanner @@ -150,7 +175,7 @@ func TestNextValueBig(t *testing.T) { t.Errorf("invalid rest: %d", len(rest)) } - item, rest, err = nextValue(append(jsonBig, []byte("HELLO WORLD")...), &scan) + item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan) if err != nil { t.Fatalf("nextValue extra: %s", err) } @@ -235,10 +260,10 @@ func genValue(n int) interface{} { } func genString(stddev float64) string { - n := int(math.Fabs(rand.NormFloat64()*stddev + stddev/2)) + n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2)) c := make([]int, n) for i := range c { - f := math.Fabs(rand.NormFloat64()*64 + 32) + f := math.Abs(rand.NormFloat64()*64 + 32) if f > 0x10ffff { f = 0x10ffff } @@ -248,7 +273,7 @@ func genString(stddev float64) string { } func genArray(n int) []interface{} { - f := int(math.Fabs(rand.NormFloat64()) * math.Fmin(10, float64(n/2))) + f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) if f > n { f = n } @@ -263,7 +288,7 @@ func genArray(n int) []interface{} { } func genMap(n int) map[string]interface{} { - f := int(math.Fabs(rand.NormFloat64()) * math.Fmin(10, float64(n/2))) + f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) if f > n { f = n } diff --git a/libgo/go/json/stream_test.go b/libgo/go/json/stream_test.go index 6ddaed9..ce5a7e6 100644 --- a/libgo/go/json/stream_test.go +++ b/libgo/go/json/stream_test.go @@ -120,3 +120,28 @@ func TestRawMessage(t *testing.T) { t.Fatalf("Marshal: have %#q want %#q", b, msg) } } + +func TestNullRawMessage(t *testing.T) { + // TODO(rsc): Should not need the * in *RawMessage + var data struct { + X float64 + Id *RawMessage + Y float32 + } + data.Id = new(RawMessage) + const msg = `{"X":0.1,"Id":null,"Y":0.2}` + err := Unmarshal([]byte(msg), &data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if data.Id != nil { + t.Fatalf("Raw mismatch: have non-nil, want nil") + } + b, err := Marshal(&data) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(b) != msg { + t.Fatalf("Marshal: have %#q want %#q", b, msg) + } +} diff --git a/libgo/go/mail/message_test.go b/libgo/go/mail/message_test.go index e1bcc89..5653647 100644 --- a/libgo/go/mail/message_test.go +++ b/libgo/go/mail/message_test.go @@ -94,7 +94,6 @@ func TestDateParsing(t *testing.T) { Hour: 9, Minute: 55, Second: 6, - Weekday: 5, // Fri ZoneOffset: -6 * 60 * 60, }, }, diff --git a/libgo/go/math/fabs.go b/libgo/go/math/abs.go index 3431231..eb3e4c7 100644 --- a/libgo/go/math/fabs.go +++ b/libgo/go/math/abs.go @@ -4,18 +4,18 @@ package math -// Fabs returns the absolute value of x. +// Abs returns the absolute value of x. // // Special cases are: -// Fabs(+Inf) = +Inf -// Fabs(-Inf) = +Inf -// Fabs(NaN) = NaN -func Fabs(x float64) float64 { +// Abs(+Inf) = +Inf +// Abs(-Inf) = +Inf +// Abs(NaN) = NaN +func Abs(x float64) float64 { switch { case x < 0: return -x case x == 0: - return 0 // return correctly fabs(-0) + return 0 // return correctly abs(-0) } return x } diff --git a/libgo/go/math/abs_decl.go b/libgo/go/math/abs_decl.go new file mode 100644 index 0000000..6be9305 --- /dev/null +++ b/libgo/go/math/abs_decl.go @@ -0,0 +1,7 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package math + +func Abs(x float64) float64 diff --git a/libgo/go/math/all_test.go b/libgo/go/math/all_test.go index d2a7d41..94ddea2 100644 --- a/libgo/go/math/all_test.go +++ b/libgo/go/math/all_test.go @@ -1359,6 +1359,20 @@ var powSC = []float64{ NaN(), // pow(NaN, NaN) } +var vfpow10SC = []int{ + MinInt32, + MaxInt32, + -325, + 309, +} + +var pow10SC = []float64{ + 0, // pow10(MinInt32) + Inf(1), // pow10(MaxInt32) + 0, // pow10(-325) + Inf(1), // pow10(309) +} + var vfsignbitSC = []float64{ Inf(-1), Copysign(0, -1), @@ -1570,7 +1584,7 @@ func TestAcos(t *testing.T) { func TestAcosh(t *testing.T) { for i := 0; i < len(vf); i++ { - a := 1 + Fabs(vf[i]) + a := 1 + Abs(vf[i]) if f := Acosh(a); !veryclose(acosh[i], f) { t.Errorf("Acosh(%g) = %g, want %g", a, f, acosh[i]) } @@ -1804,23 +1818,23 @@ func testExp2(t *testing.T, Exp2 func(float64) float64, name string) { } } -func TestFabs(t *testing.T) { +func TestAbs(t *testing.T) { for i := 0; i < len(vf); i++ { - if f := Fabs(vf[i]); fabs[i] != f { - t.Errorf("Fabs(%g) = %g, want %g", vf[i], f, fabs[i]) + if f := Abs(vf[i]); fabs[i] != f { + t.Errorf("Abs(%g) = %g, want %g", vf[i], f, fabs[i]) } } for i := 0; i < len(vffabsSC); i++ { - if f := Fabs(vffabsSC[i]); !alike(fabsSC[i], f) { - t.Errorf("Fabs(%g) = %g, want %g", vffabsSC[i], f, fabsSC[i]) + if f := Abs(vffabsSC[i]); !alike(fabsSC[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vffabsSC[i], f, fabsSC[i]) } } } -func TestFdim(t *testing.T) { +func TestDim(t *testing.T) { for i := 0; i < len(vf); i++ { - if f := Fdim(vf[i], 0); fdim[i] != f { - t.Errorf("Fdim(%g, %g) = %g, want %g", vf[i], 0.0, f, fdim[i]) + if f := Dim(vf[i], 0); fdim[i] != f { + t.Errorf("Dim(%g, %g) = %g, want %g", vf[i], 0.0, f, fdim[i]) } } } @@ -1838,31 +1852,31 @@ func TestFloor(t *testing.T) { } } -func TestFmax(t *testing.T) { +func TestMax(t *testing.T) { for i := 0; i < len(vf); i++ { - if f := Fmax(vf[i], ceil[i]); ceil[i] != f { - t.Errorf("Fmax(%g, %g) = %g, want %g", vf[i], ceil[i], f, ceil[i]) + if f := Max(vf[i], ceil[i]); ceil[i] != f { + t.Errorf("Max(%g, %g) = %g, want %g", vf[i], ceil[i], f, ceil[i]) } } } -func TestFmin(t *testing.T) { +func TestMin(t *testing.T) { for i := 0; i < len(vf); i++ { - if f := Fmin(vf[i], floor[i]); floor[i] != f { - t.Errorf("Fmin(%g, %g) = %g, want %g", vf[i], floor[i], f, floor[i]) + if f := Min(vf[i], floor[i]); floor[i] != f { + t.Errorf("Min(%g, %g) = %g, want %g", vf[i], floor[i], f, floor[i]) } } } -func TestFmod(t *testing.T) { +func TestMod(t *testing.T) { for i := 0; i < len(vf); i++ { - if f := Fmod(10, vf[i]); fmod[i] != f { - t.Errorf("Fmod(10, %g) = %g, want %g", vf[i], f, fmod[i]) + if f := Mod(10, vf[i]); fmod[i] != f { + t.Errorf("Mod(10, %g) = %g, want %g", vf[i], f, fmod[i]) } } for i := 0; i < len(vffmodSC); i++ { - if f := Fmod(vffmodSC[i][0], vffmodSC[i][1]); !alike(fmodSC[i], f) { - t.Errorf("Fmod(%g, %g) = %g, want %g", vffmodSC[i][0], vffmodSC[i][1], f, fmodSC[i]) + if f := Mod(vffmodSC[i][0], vffmodSC[i][1]); !alike(fmodSC[i], f) { + t.Errorf("Mod(%g, %g) = %g, want %g", vffmodSC[i][0], vffmodSC[i][1], f, fmodSC[i]) } } } @@ -1900,7 +1914,7 @@ func TestGamma(t *testing.T) { func TestHypot(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(1e200 * tanh[i] * Sqrt(2)) + a := Abs(1e200 * tanh[i] * Sqrt(2)) if f := Hypot(1e200*tanh[i], 1e200*tanh[i]); !veryclose(a, f) { t.Errorf("Hypot(%g, %g) = %g, want %g", 1e200*tanh[i], 1e200*tanh[i], f, a) } @@ -2019,7 +2033,7 @@ func TestLgamma(t *testing.T) { func TestLog(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := Log(a); log[i] != f { t.Errorf("Log(%g) = %g, want %g", a, f, log[i]) } @@ -2054,7 +2068,7 @@ func TestLogb(t *testing.T) { func TestLog10(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := Log10(a); !veryclose(log10[i], f) { t.Errorf("Log10(%g) = %g, want %g", a, f, log10[i]) } @@ -2089,7 +2103,7 @@ func TestLog1p(t *testing.T) { func TestLog2(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := Log2(a); !veryclose(log2[i], f) { t.Errorf("Log2(%g) = %g, want %g", a, f, log2[i]) } @@ -2143,6 +2157,14 @@ func TestPow(t *testing.T) { } } +func TestPow10(t *testing.T) { + for i := 0; i < len(vfpow10SC); i++ { + if f := Pow10(vfpow10SC[i]); !alike(pow10SC[i], f) { + t.Errorf("Pow10(%d) = %g, want %g", vfpow10SC[i], f, pow10SC[i]) + } + } +} + func TestRemainder(t *testing.T) { for i := 0; i < len(vf); i++ { if f := Remainder(10, vf[i]); remainder[i] != f { @@ -2204,11 +2226,11 @@ func TestSinh(t *testing.T) { func TestSqrt(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := SqrtGo(a); sqrt[i] != f { t.Errorf("SqrtGo(%g) = %g, want %g", a, f, sqrt[i]) } - a = Fabs(vf[i]) + a = Abs(vf[i]) if f := Sqrt(a); sqrt[i] != f { t.Errorf("Sqrt(%g) = %g, want %g", a, f, sqrt[i]) } @@ -2275,7 +2297,7 @@ func TestTrunc(t *testing.T) { func TestY0(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := Y0(a); !close(y0[i], f) { t.Errorf("Y0(%g) = %g, want %g", a, f, y0[i]) } @@ -2289,7 +2311,7 @@ func TestY0(t *testing.T) { func TestY1(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := Y1(a); !soclose(y1[i], f, 2e-14) { t.Errorf("Y1(%g) = %g, want %g", a, f, y1[i]) } @@ -2303,7 +2325,7 @@ func TestY1(t *testing.T) { func TestYn(t *testing.T) { for i := 0; i < len(vf); i++ { - a := Fabs(vf[i]) + a := Abs(vf[i]) if f := Yn(2, a); !close(y2[i], f) { t.Errorf("Yn(2, %g) = %g, want %g", a, f, y2[i]) } @@ -2509,15 +2531,15 @@ func BenchmarkExp2Go(b *testing.B) { } } -func BenchmarkFabs(b *testing.B) { +func BenchmarkAbs(b *testing.B) { for i := 0; i < b.N; i++ { - Fabs(.5) + Abs(.5) } } -func BenchmarkFdim(b *testing.B) { +func BenchmarkDim(b *testing.B) { for i := 0; i < b.N; i++ { - Fdim(10, 3) + Dim(10, 3) } } @@ -2527,21 +2549,21 @@ func BenchmarkFloor(b *testing.B) { } } -func BenchmarkFmax(b *testing.B) { +func BenchmarkMax(b *testing.B) { for i := 0; i < b.N; i++ { - Fmax(10, 3) + Max(10, 3) } } -func BenchmarkFmin(b *testing.B) { +func BenchmarkMin(b *testing.B) { for i := 0; i < b.N; i++ { - Fmin(10, 3) + Min(10, 3) } } -func BenchmarkFmod(b *testing.B) { +func BenchmarkMod(b *testing.B) { for i := 0; i < b.N; i++ { - Fmod(10, 3) + Mod(10, 3) } } @@ -2659,6 +2681,18 @@ func BenchmarkPowFrac(b *testing.B) { } } +func BenchmarkPow10Pos(b *testing.B) { + for i := 0; i < b.N; i++ { + Pow10(300) + } +} + +func BenchmarkPow10Neg(b *testing.B) { + for i := 0; i < b.N; i++ { + Pow10(-300) + } +} + func BenchmarkRemainder(b *testing.B) { for i := 0; i < b.N; i++ { Remainder(10, 3) diff --git a/libgo/go/math/bits.go b/libgo/go/math/bits.go index a1dca3e..1cf60ce 100644 --- a/libgo/go/math/bits.go +++ b/libgo/go/math/bits.go @@ -52,7 +52,7 @@ func IsInf(f float64, sign int) bool { // satisfying x == y × 2**exp. It assumes x is finite and non-zero. func normalize(x float64) (y float64, exp int) { const SmallestNormal = 2.2250738585072014e-308 // 2**-1022 - if Fabs(x) < SmallestNormal { + if Abs(x) < SmallestNormal { return x * (1 << 52), -52 } return x, 0 diff --git a/libgo/go/math/fdim.go b/libgo/go/math/dim.go index 1899313..d2eb52f 100644 --- a/libgo/go/math/fdim.go +++ b/libgo/go/math/dim.go @@ -4,24 +4,24 @@ package math -// Fdim returns the maximum of x-y or 0. -func Fdim(x, y float64) float64 { +// Dim returns the maximum of x-y or 0. +func Dim(x, y float64) float64 { if x > y { return x - y } return 0 } -// Fmax returns the larger of x or y. -func Fmax(x, y float64) float64 { +// Max returns the larger of x or y. +func Max(x, y float64) float64 { if x > y { return x } return y } -// Fmin returns the smaller of x or y. -func Fmin(x, y float64) float64 { +// Min returns the smaller of x or y. +func Min(x, y float64) float64 { if x < y { return x } diff --git a/libgo/go/math/dim_decl.go b/libgo/go/math/dim_decl.go new file mode 100644 index 0000000..196f84f --- /dev/null +++ b/libgo/go/math/dim_decl.go @@ -0,0 +1,9 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package math + +func Dim(x, y float64) float64 +func Max(x, y float64) float64 +func Min(x, y float64) float64 diff --git a/libgo/go/math/gamma.go b/libgo/go/math/gamma.go index 73ca0e5..0136507 100644 --- a/libgo/go/math/gamma.go +++ b/libgo/go/math/gamma.go @@ -127,7 +127,7 @@ func Gamma(x float64) float64 { case x < -170.5674972726612 || x > 171.61447887182298: return Inf(1) } - q := Fabs(x) + q := Abs(x) p := Floor(q) if q > 33 { if x >= 0 { @@ -146,7 +146,7 @@ func Gamma(x float64) float64 { if z == 0 { return Inf(signgam) } - z = Pi / (Fabs(z) * stirling(q)) + z = Pi / (Abs(z) * stirling(q)) return float64(signgam) * z } diff --git a/libgo/go/math/jn.go b/libgo/go/math/jn.go index 9024af3..1878df5 100644 --- a/libgo/go/math/jn.go +++ b/libgo/go/math/jn.go @@ -197,7 +197,7 @@ func Jn(n int, x float64) float64 { tmp := float64(n) v := 2 / x - tmp = tmp * Log(Fabs(v*tmp)) + tmp = tmp * Log(Abs(v*tmp)) if tmp < 7.09782712893383973096e+02 { for i := n - 1; i > 0; i-- { di := float64(i + i) diff --git a/libgo/go/math/lgamma.go b/libgo/go/math/lgamma.go index dc30f46..8f6d7b9 100644 --- a/libgo/go/math/lgamma.go +++ b/libgo/go/math/lgamma.go @@ -206,7 +206,7 @@ func Lgamma(x float64) (lgamma float64, sign int) { lgamma = Inf(1) // -integer return } - nadj = Log(Pi / Fabs(t*x)) + nadj = Log(Pi / Abs(t*x)) if t < 0 { sign = -1 } @@ -319,7 +319,7 @@ func sinPi(x float64) float64 { z := Floor(x) var n int if z != x { // inexact - x = Fmod(x, 2) + x = Mod(x, 2) n = int(x * 4) } else { if x >= Two53 { // x must be even diff --git a/libgo/go/math/fmod.go b/libgo/go/math/mod.go index 75c6146..6b16abe 100644 --- a/libgo/go/math/fmod.go +++ b/libgo/go/math/mod.go @@ -8,14 +8,14 @@ package math Floating-point mod function. */ -// Fmod returns the floating-point remainder of x/y. +// Mod returns the floating-point remainder of x/y. // The magnitude of the result is less than y and its // sign agrees with that of x. // // Special cases are: -// if x is not finite, Fmod returns NaN -// if y is 0 or NaN, Fmod returns NaN -func Fmod(x, y float64) float64 { +// if x is not finite, Mod returns NaN +// if y is 0 or NaN, Mod returns NaN +func Mod(x, y float64) float64 { // TODO(rsc): Remove manual inlining of IsNaN, IsInf // when compiler does it for us. if y == 0 || x > MaxFloat64 || x < -MaxFloat64 || x != x || y != y { // y == 0 || IsInf(x, 0) || IsNaN(x) || IsNan(y) diff --git a/libgo/go/math/mod_decl.go b/libgo/go/math/mod_decl.go new file mode 100644 index 0000000..d5047a7 --- /dev/null +++ b/libgo/go/math/mod_decl.go @@ -0,0 +1,7 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package math + +func Mod(x, y float64) float64 diff --git a/libgo/go/math/pow.go b/libgo/go/math/pow.go index 06b1074..f0f52c5 100644 --- a/libgo/go/math/pow.go +++ b/libgo/go/math/pow.go @@ -66,7 +66,7 @@ func Pow(x, y float64) float64 { switch { case x == -1: return 1 - case (Fabs(x) < 1) == IsInf(y, 1): + case (Abs(x) < 1) == IsInf(y, 1): return 0 default: return Inf(1) diff --git a/libgo/go/math/pow10.go b/libgo/go/math/pow10.go index bda2e82..20f91bc 100644 --- a/libgo/go/math/pow10.go +++ b/libgo/go/math/pow10.go @@ -10,6 +10,12 @@ var pow10tab [70]float64 // Pow10 returns 10**e, the base-10 exponential of e. func Pow10(e int) float64 { + if e <= -325 { + return 0 + } else if e > 309 { + return Inf(1) + } + if e < 0 { return 1 / Pow10(-e) } diff --git a/libgo/go/math/remainder.go b/libgo/go/math/remainder.go index be8724c..7fb8a12 100644 --- a/libgo/go/math/remainder.go +++ b/libgo/go/math/remainder.go @@ -24,7 +24,7 @@ package math // precision arithmetic, where [x/y] is the (infinite bit) // integer nearest x/y (in half way cases, choose the even one). // Method : -// Based on fmod() returning x - [x/y]chopped * y exactly. +// Based on Mod() returning x - [x/y]chopped * y exactly. // Remainder returns the IEEE 754 floating-point remainder of x/y. // @@ -60,7 +60,7 @@ func Remainder(x, y float64) float64 { return 0 } if y <= HalfMax { - x = Fmod(x, y+y) // now x < 2y + x = Mod(x, y+y) // now x < 2y } if y < Tiny { if x+x > y { diff --git a/libgo/go/mime/grammar.go b/libgo/go/mime/grammar.go index 6e319ff..70a94cd 100644 --- a/libgo/go/mime/grammar.go +++ b/libgo/go/mime/grammar.go @@ -22,6 +22,15 @@ func IsTokenChar(rune int) bool { return rune > 0x20 && rune < 0x7f && !isTSpecial(rune) } +// IsToken returns true if s is a 'token' as as defined by RFC 1521 +// and RFC 2045. +func IsToken(s string) bool { + if s == "" { + return false + } + return strings.IndexFunc(s, isNotTokenChar) < 0 +} + // IsQText returns true if rune is in 'qtext' as defined by RFC 822. func IsQText(rune int) bool { // CHAR = <any ASCII character> ; ( 0-177, 0.-127.) diff --git a/libgo/go/mime/mediatype.go b/libgo/go/mime/mediatype.go index 40c735c..b0d3933 100644 --- a/libgo/go/mime/mediatype.go +++ b/libgo/go/mime/mediatype.go @@ -12,40 +12,89 @@ import ( "unicode" ) -func validMediaTypeOrDisposition(s string) bool { +// FormatMediaType serializes type t, subtype sub and the paramaters +// param as a media type conform RFC 2045 and RFC 2616. +// The type, subtype, and parameter names are written in lower-case. +// When any of the arguments result in a standard violation then +// FormatMediaType returns the empty string. +func FormatMediaType(t, sub string, param map[string]string) string { + if !(IsToken(t) && IsToken(sub)) { + return "" + } + var b bytes.Buffer + b.WriteString(strings.ToLower(t)) + b.WriteByte('/') + b.WriteString(strings.ToLower(sub)) + + for attribute, value := range param { + b.WriteByte(';') + b.WriteByte(' ') + if !IsToken(attribute) { + return "" + } + b.WriteString(strings.ToLower(attribute)) + b.WriteByte('=') + if IsToken(value) { + b.WriteString(value) + continue + } + + b.WriteByte('"') + offset := 0 + for index, character := range value { + if character == '"' || character == '\r' { + b.WriteString(value[offset:index]) + offset = index + b.WriteByte('\\') + } + if character&0x80 != 0 { + return "" + } + } + b.WriteString(value[offset:]) + b.WriteByte('"') + } + return b.String() +} + +func checkMediaTypeDisposition(s string) os.Error { typ, rest := consumeToken(s) if typ == "" { - return false + return os.NewError("mime: no media type") } if rest == "" { - return true + return nil } if !strings.HasPrefix(rest, "/") { - return false + return os.NewError("mime: expected slash after first token") } subtype, rest := consumeToken(rest[1:]) if subtype == "" { - return false + return os.NewError("mime: expected token after slash") } - return rest == "" + if rest != "" { + return os.NewError("mime: unexpected content after media subtype") + } + return nil } // ParseMediaType parses a media type value and any optional // parameters, per RFC 1521. Media types are the values in // Content-Type and Content-Disposition headers (RFC 2183). // On success, ParseMediaType returns the media type converted -// to lowercase and trimmed of white space. The returned params -// is always a non-nil map. Params maps from the lowercase +// to lowercase and trimmed of white space and a non-nil map. +// The returned map, params, maps from the lowercase // attribute to the attribute value with its case preserved. -// On error, it returns an empty string and a nil params. -func ParseMediaType(v string) (mediatype string, params map[string]string) { +func ParseMediaType(v string) (mediatype string, params map[string]string, err os.Error) { i := strings.Index(v, ";") if i == -1 { i = len(v) } mediatype = strings.TrimSpace(strings.ToLower(v[0:i])) - if !validMediaTypeOrDisposition(mediatype) { - return "", nil + + err = checkMediaTypeDisposition(mediatype) + if err != nil { + return } params = make(map[string]string) @@ -69,7 +118,7 @@ func ParseMediaType(v string) (mediatype string, params map[string]string) { return } // Parse error. - return "", nil + return "", nil, os.NewError("mime: invalid media parameter") } pmap := params @@ -86,7 +135,7 @@ func ParseMediaType(v string) (mediatype string, params map[string]string) { } if _, exists := pmap[key]; exists { // Duplicate parameter name is bogus. - return "", nil + return "", nil, os.NewError("mime: duplicate parameter name") } pmap[key] = value v = rest diff --git a/libgo/go/mime/mediatype_test.go b/libgo/go/mime/mediatype_test.go index 93264bd..884573e 100644 --- a/libgo/go/mime/mediatype_test.go +++ b/libgo/go/mime/mediatype_test.go @@ -219,7 +219,14 @@ func TestParseMediaType(t *testing.T) { m("firstname", "Брэд", "lastname", "Фицпатрик")}, } for _, test := range tests { - mt, params := ParseMediaType(test.in) + mt, params, err := ParseMediaType(test.in) + if err != nil { + if test.t != "" { + t.Errorf("for input %q, unexpected error: %v", test.in, err) + continue + } + continue + } if g, e := mt, test.t; g != e { t.Errorf("for input %q, expected type %q, got %q", test.in, e, g) @@ -238,11 +245,11 @@ func TestParseMediaType(t *testing.T) { } func TestParseMediaTypeBogus(t *testing.T) { - mt, params := ParseMediaType("bogus ;=========") - if mt != "" { - t.Error("expected empty type") + mt, params, err := ParseMediaType("bogus ;=========") + if err == nil { + t.Fatalf("expected an error parsing invalid media type; got type %q, params %#v", mt, params) } - if params != nil { - t.Error("expected nil params") + if err.String() != "mime: invalid media parameter" { + t.Errorf("expected invalid media parameter; got error %q", err) } } diff --git a/libgo/go/mime/multipart/formdata.go b/libgo/go/mime/multipart/formdata.go index 91404d6f..d114bfa 100644 --- a/libgo/go/mime/multipart/formdata.go +++ b/libgo/go/mime/multipart/formdata.go @@ -47,7 +47,7 @@ func (r *Reader) ReadForm(maxMemory int64) (f *Form, err os.Error) { if filename == "" { // value, store as string in memory - n, err := io.Copyn(&b, p, maxValueBytes) + n, err := io.CopyN(&b, p, maxValueBytes) if err != nil && err != os.EOF { return nil, err } @@ -64,7 +64,7 @@ func (r *Reader) ReadForm(maxMemory int64) (f *Form, err os.Error) { Filename: filename, Header: p.Header, } - n, err := io.Copyn(&b, p, maxMemory+1) + n, err := io.CopyN(&b, p, maxMemory+1) if err != nil && err != os.EOF { return nil, err } diff --git a/libgo/go/mime/multipart/multipart.go b/libgo/go/mime/multipart/multipart.go index 2533bd3..d36e9e9 100644 --- a/libgo/go/mime/multipart/multipart.go +++ b/libgo/go/mime/multipart/multipart.go @@ -69,8 +69,9 @@ func (p *Part) FileName() string { func (p *Part) parseContentDisposition() { v := p.Header.Get("Content-Disposition") - p.disposition, p.dispositionParams = mime.ParseMediaType(v) - if p.dispositionParams == nil { + var err os.Error + p.disposition, p.dispositionParams, err = mime.ParseMediaType(v) + if err != nil { p.dispositionParams = emptyParams } } @@ -145,7 +146,7 @@ func (bp *Part) Read(p []byte) (n int, err os.Error) { return 0, io.ErrUnexpectedEOF } if nCopy > 0 { - if _, err := io.Copyn(bp.buffer, bp.mr.bufReader, int64(nCopy)); err != nil { + if _, err := io.CopyN(bp.buffer, bp.mr.bufReader, int64(nCopy)); err != nil { return 0, err } } diff --git a/libgo/go/mime/multipart/writer.go b/libgo/go/mime/multipart/writer.go index 97a8897..1bff02f 100644 --- a/libgo/go/mime/multipart/writer.go +++ b/libgo/go/mime/multipart/writer.go @@ -85,10 +85,10 @@ func (w *Writer) CreatePart(header textproto.MIMEHeader) (io.Writer, os.Error) { return p, nil } +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + func escapeQuotes(s string) string { - s = strings.Replace(s, "\\", "\\\\", -1) - s = strings.Replace(s, "\"", "\\\"", -1) - return s + return quoteEscaper.Replace(s) } // CreateFormFile is a convenience wrapper around CreatePart. It creates diff --git a/libgo/go/mime/type.go b/libgo/go/mime/type.go index 8ecfe9a..39bf40e 100644 --- a/libgo/go/mime/type.go +++ b/libgo/go/mime/type.go @@ -7,6 +7,7 @@ package mime import ( "bufio" + "fmt" "os" "strings" "sync" @@ -49,15 +50,12 @@ func loadMimeFile(filename string) { if len(fields) <= 1 || fields[0][0] == '#' { continue } - typename := fields[0] - if strings.HasPrefix(typename, "text/") { - typename += "; charset=utf-8" - } + mimeType := fields[0] for _, ext := range fields[1:] { if ext[0] == '#' { break } - mimeTypes["."+ext] = typename + setExtensionType("."+ext, mimeType) } } } @@ -81,6 +79,8 @@ var once sync.Once // /etc/mime.types // /etc/apache2/mime.types // /etc/apache/mime.types +// +// Text types have the charset parameter set to "utf-8" by default. func TypeByExtension(ext string) string { once.Do(initMime) mimeLock.RLock() @@ -93,12 +93,31 @@ func TypeByExtension(ext string) string { // the extension ext to typ. The extension should begin with // a leading dot, as in ".html". func AddExtensionType(ext, typ string) os.Error { + if ext == "" || ext[0] != '.' { + return fmt.Errorf(`mime: extension "%s" misses dot`, ext) + } once.Do(initMime) - if len(ext) < 1 || ext[0] != '.' { - return os.EINVAL + return setExtensionType(ext, typ) +} + +func setExtensionType(extension, mimeType string) os.Error { + full, param, err := ParseMediaType(mimeType) + if err != nil { + return err + } + if split := strings.Index(full, "/"); split < 0 { + return fmt.Errorf(`mime: malformed MIME type "%s"`, mimeType) + } else { + main := full[:split] + sub := full[split+1:] + if main == "text" && param["charset"] == "" { + param["charset"] = "utf-8" + } + mimeType = FormatMediaType(main, sub, param) } + mimeLock.Lock() - mimeTypes[ext] = typ + mimeTypes[extension] = mimeType mimeLock.Unlock() return nil } diff --git a/libgo/go/mime/mime_test.go b/libgo/go/mime/type_test.go index 17e6104..976f853 100644 --- a/libgo/go/mime/mime_test.go +++ b/libgo/go/mime/type_test.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Tests for type.go - package mime import "testing" @@ -14,7 +12,7 @@ var typeTests = map[string]string{ ".png": "image/png", } -func TestType(t *testing.T) { +func TestTypeByExtension(t *testing.T) { typeFiles = []string{"test.types"} for ext, want := range typeTests { @@ -25,3 +23,13 @@ func TestType(t *testing.T) { } } + +func TestCustomExtension(t *testing.T) { + custom := "text/xml; charset=iso-8859-1" + if error := AddExtensionType(".xml", custom); error != nil { + t.Fatalf("error %s for AddExtension(%s)", error, custom) + } + if registered := TypeByExtension(".xml"); registered != custom { + t.Fatalf("registered %s instead of %s", registered, custom) + } +} diff --git a/libgo/go/net/cgo_bsd.go b/libgo/go/net/cgo_bsd.go index 3951d84..b8c6ef9 100644 --- a/libgo/go/net/cgo_bsd.go +++ b/libgo/go/net/cgo_bsd.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd + package net /* diff --git a/libgo/go/net/cgo_stub.go b/libgo/go/net/cgo_stub.go index c6277cb..565cbe7 100644 --- a/libgo/go/net/cgo_stub.go +++ b/libgo/go/net/cgo_stub.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build openbsd + // Stub cgo routines for systems that do not use cgo to do network lookups. package net diff --git a/libgo/go/net/cgo_unix.go b/libgo/go/net/cgo_unix.go index d088eab..bfb3dfd 100644 --- a/libgo/go/net/cgo_unix.go +++ b/libgo/go/net/cgo_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux + package net /* diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 10c67dc..85d54b3 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -59,10 +59,10 @@ func Dial(net, addr string) (c Conn, err os.Error) { case *IPAddr: c, err = DialIP(net, nil, ra) default: - err = UnknownNetworkError(net) + err = &OpError{"dial", net + " " + addr, nil, UnknownNetworkError(net)} } if err != nil { - return nil, &OpError{"dial", net + " " + addr, nil, err} + return nil, err } return } diff --git a/libgo/go/net/dnsclient_unix.go b/libgo/go/net/dnsclient_unix.go index f407b17..eb7db5e 100644 --- a/libgo/go/net/dnsclient_unix.go +++ b/libgo/go/net/dnsclient_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + // DNS client: see RFC 1035. // Has to be linked into package net for Dial. @@ -113,7 +115,7 @@ func convertRR_A(records []dnsRR) []IP { func convertRR_AAAA(records []dnsRR) []IP { addrs := make([]IP, len(records)) for i, rr := range records { - a := make(IP, 16) + a := make(IP, IPv6len) copy(a, rr.(*dnsRR_AAAA).AAAA[:]) addrs[i] = a } @@ -213,6 +215,18 @@ func goLookupHost(name string) (addrs []string, err os.Error) { // depending on our lookup code, so that Go and C get the same // answers. func goLookupIP(name string) (addrs []IP, err os.Error) { + // Use entries from /etc/hosts if possible. + haddrs := lookupStaticHost(name) + if len(haddrs) > 0 { + for _, haddr := range haddrs { + if ip := ParseIP(haddr); ip != nil { + addrs = append(addrs, ip) + } + } + if len(addrs) > 0 { + return + } + } onceLoadConfig.Do(loadConfig) if dnserr != nil || cfg == nil { err = dnserr diff --git a/libgo/go/net/dnsconfig.go b/libgo/go/net/dnsconfig.go index 54e3343..afc0599 100644 --- a/libgo/go/net/dnsconfig.go +++ b/libgo/go/net/dnsconfig.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + // Read system DNS config from /etc/resolv.conf package net diff --git a/libgo/go/net/fd.go b/libgo/go/net/fd.go index 707dcca..80d40af 100644 --- a/libgo/go/net/fd.go +++ b/libgo/go/net/fd.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package net import ( @@ -150,7 +152,7 @@ func (s *pollServer) LookupFD(fd int, mode int) *netFD { if !ok { return nil } - s.pending[key] = nil, false + delete(s.pending, key) return netfd } @@ -193,7 +195,7 @@ func (s *pollServer) CheckDeadlines() { } if t > 0 { if t <= now { - s.pending[key] = nil, false + delete(s.pending, key) if mode == 'r' { s.poll.DelFD(fd.sysfd, mode) fd.rdeadline = -1 @@ -356,6 +358,25 @@ func (fd *netFD) Close() os.Error { return nil } +func (fd *netFD) shutdown(how int) os.Error { + if fd == nil || fd.sysfile == nil { + return os.EINVAL + } + errno := syscall.Shutdown(fd.sysfd, how) + if errno != 0 { + return &OpError{"shutdown", fd.net, fd.laddr, os.Errno(errno)} + } + return nil +} + +func (fd *netFD) CloseRead() os.Error { + return fd.shutdown(syscall.SHUT_RD) +} + +func (fd *netFD) CloseWrite() os.Error { + return fd.shutdown(syscall.SHUT_WR) +} + func (fd *netFD) Read(p []byte) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL diff --git a/libgo/go/net/fd_linux.go b/libgo/go/net/fd_linux.go index 70fc344b..c860c84 100644 --- a/libgo/go/net/fd_linux.go +++ b/libgo/go/net/fd_linux.go @@ -105,7 +105,7 @@ func (p *pollster) StopWaiting(fd int, bits uint) { if e := syscall.EpollCtl(p.epfd, syscall.EPOLL_CTL_DEL, fd, nil); e != 0 { print("Epoll delete fd=", fd, ": ", os.Errno(e).String(), "\n") } - p.events[fd] = 0, false + delete(p.events, fd) } } diff --git a/libgo/go/net/fd_windows.go b/libgo/go/net/fd_windows.go index 3757e14..8e8b3b7 100644 --- a/libgo/go/net/fd_windows.go +++ b/libgo/go/net/fd_windows.go @@ -23,7 +23,7 @@ var initErr os.Error func init() { var d syscall.WSAData - e := syscall.WSAStartup(uint32(0x101), &d) + e := syscall.WSAStartup(uint32(0x202), &d) if e != 0 { initErr = os.NewSyscallError("WSAStartup", e) } @@ -52,15 +52,27 @@ type anOp struct { // of the struct, as our code rely on it. o syscall.Overlapped - resultc chan ioResult // io completion results - errnoc chan int // io submit / cancel operation errors + resultc chan ioResult + errnoc chan int fd *netFD } -func (o *anOp) Init(fd *netFD) { +func (o *anOp) Init(fd *netFD, mode int) { o.fd = fd - o.resultc = make(chan ioResult, 1) - o.errnoc = make(chan int) + var i int + if mode == 'r' { + i = 0 + } else { + i = 1 + } + if fd.resultc[i] == nil { + fd.resultc[i] = make(chan ioResult, 1) + } + o.resultc = fd.resultc[i] + if fd.errnoc[i] == nil { + fd.errnoc[i] = make(chan int) + } + o.errnoc = fd.errnoc[i] } func (o *anOp) Op() *anOp { @@ -74,8 +86,8 @@ type bufOp struct { buf syscall.WSABuf } -func (o *bufOp) Init(fd *netFD, buf []byte) { - o.anOp.Init(fd) +func (o *bufOp) Init(fd *netFD, buf []byte, mode int) { + o.anOp.Init(fd, mode) o.buf.Len = uint32(len(buf)) if len(buf) == 0 { o.buf.Buf = nil @@ -208,12 +220,14 @@ type netFD struct { closing bool // immutable until Close - sysfd syscall.Handle - family int - proto int - net string - laddr Addr - raddr Addr + sysfd syscall.Handle + family int + proto int + net string + laddr Addr + raddr Addr + resultc [2]chan ioResult // read/write completion results + errnoc [2]chan int // read/write submit or cancel operation errors // owned by client rdeadline_delta int64 @@ -298,6 +312,25 @@ func (fd *netFD) Close() os.Error { return nil } +func (fd *netFD) shutdown(how int) os.Error { + if fd == nil || fd.sysfd == syscall.InvalidHandle { + return os.EINVAL + } + errno := syscall.Shutdown(fd.sysfd, how) + if errno != 0 { + return &OpError{"shutdown", fd.net, fd.laddr, os.Errno(errno)} + } + return nil +} + +func (fd *netFD) CloseRead() os.Error { + return fd.shutdown(syscall.SHUT_RD) +} + +func (fd *netFD) CloseWrite() os.Error { + return fd.shutdown(syscall.SHUT_WR) +} + // Read from network. type readOp struct { @@ -325,7 +358,7 @@ func (fd *netFD) Read(buf []byte) (n int, err os.Error) { return 0, os.EINVAL } var o readOp - o.Init(fd, buf) + o.Init(fd, buf, 'r') n, err = iosrv.ExecIO(&o, fd.rdeadline_delta) if err == nil && n == 0 { err = os.EOF @@ -365,7 +398,7 @@ func (fd *netFD) ReadFrom(buf []byte) (n int, sa syscall.Sockaddr, err os.Error) return 0, nil, os.EINVAL } var o readFromOp - o.Init(fd, buf) + o.Init(fd, buf, 'r') o.rsan = int32(unsafe.Sizeof(o.rsa)) n, err = iosrv.ExecIO(&o, fd.rdeadline_delta) if err != nil { @@ -402,7 +435,7 @@ func (fd *netFD) Write(buf []byte) (n int, err os.Error) { return 0, os.EINVAL } var o writeOp - o.Init(fd, buf) + o.Init(fd, buf, 'w') return iosrv.ExecIO(&o, fd.wdeadline_delta) } @@ -437,7 +470,7 @@ func (fd *netFD) WriteTo(buf []byte, sa syscall.Sockaddr) (n int, err os.Error) return 0, os.EINVAL } var o writeToOp - o.Init(fd, buf) + o.Init(fd, buf, 'w') o.sa = sa return iosrv.ExecIO(&o, fd.wdeadline_delta) } @@ -487,7 +520,7 @@ func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os. // Submit accept request. var o acceptOp - o.Init(fd) + o.Init(fd, 'r') o.newsock = s _, err = iosrv.ExecIO(&o, 0) if err != nil { diff --git a/libgo/go/net/file.go b/libgo/go/net/file.go index 0e411a1..ed2559d 100644 --- a/libgo/go/net/file.go +++ b/libgo/go/net/file.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package net import ( @@ -20,6 +22,7 @@ func newFileFD(f *os.File) (nfd *netFD, err os.Error) { return nil, os.NewSyscallError("getsockopt", errno) } + family := syscall.AF_UNSPEC toAddr := sockaddrToTCP sa, _ := syscall.Getsockname(fd) switch sa.(type) { @@ -27,18 +30,21 @@ func newFileFD(f *os.File) (nfd *netFD, err os.Error) { closesocket(fd) return nil, os.EINVAL case *syscall.SockaddrInet4: + family = syscall.AF_INET if proto == syscall.SOCK_DGRAM { toAddr = sockaddrToUDP } else if proto == syscall.SOCK_RAW { toAddr = sockaddrToIP } case *syscall.SockaddrInet6: + family = syscall.AF_INET6 if proto == syscall.SOCK_DGRAM { toAddr = sockaddrToUDP } else if proto == syscall.SOCK_RAW { toAddr = sockaddrToIP } case *syscall.SockaddrUnix: + family = syscall.AF_UNIX toAddr = sockaddrToUnix if proto == syscall.SOCK_DGRAM { toAddr = sockaddrToUnixgram @@ -50,7 +56,7 @@ func newFileFD(f *os.File) (nfd *netFD, err os.Error) { sa, _ = syscall.Getpeername(fd) raddr := toAddr(sa) - if nfd, err = newFD(fd, 0, proto, laddr.Network()); err != nil { + if nfd, err = newFD(fd, family, proto, laddr.Network()); err != nil { return nil, err } nfd.setAddr(laddr, raddr) diff --git a/libgo/go/net/file_test.go b/libgo/go/net/file_test.go index 9a8c2dc..0fa6740 100644 --- a/libgo/go/net/file_test.go +++ b/libgo/go/net/file_test.go @@ -73,7 +73,7 @@ func TestFileListener(t *testing.T) { } } -func testFilePacketConn(t *testing.T, pcf packetConnFile) { +func testFilePacketConn(t *testing.T, pcf packetConnFile, listen bool) { f, err := pcf.File() if err != nil { t.Fatalf("File failed: %v", err) @@ -85,6 +85,11 @@ func testFilePacketConn(t *testing.T, pcf packetConnFile) { if !reflect.DeepEqual(pcf.LocalAddr(), c.LocalAddr()) { t.Fatalf("LocalAddrs not equal: %#v != %#v", pcf.LocalAddr(), c.LocalAddr()) } + if listen { + if _, err := c.WriteTo([]byte{}, c.LocalAddr()); err != nil { + t.Fatalf("WriteTo failed: %v", err) + } + } if err := c.Close(); err != nil { t.Fatalf("Close failed: %v", err) } @@ -98,7 +103,7 @@ func testFilePacketConnListen(t *testing.T, net, laddr string) { if err != nil { t.Fatalf("Listen failed: %v", err) } - testFilePacketConn(t, l.(packetConnFile)) + testFilePacketConn(t, l.(packetConnFile), true) if err := l.Close(); err != nil { t.Fatalf("Close failed: %v", err) } @@ -109,7 +114,7 @@ func testFilePacketConnDial(t *testing.T, net, raddr string) { if err != nil { t.Fatalf("Dial failed: %v", err) } - testFilePacketConn(t, c.(packetConnFile)) + testFilePacketConn(t, c.(packetConnFile), false) if err := c.Close(); err != nil { t.Fatalf("Close failed: %v", err) } diff --git a/libgo/go/net/interface.go b/libgo/go/net/interface.go index 8a14cb2..2696b7f 100644 --- a/libgo/go/net/interface.go +++ b/libgo/go/net/interface.go @@ -26,6 +26,63 @@ func (a HardwareAddr) String() string { return buf.String() } +// ParseMAC parses s as an IEEE 802 MAC-48, EUI-48, or EUI-64 using one of the +// following formats: +// 01:23:45:67:89:ab +// 01:23:45:67:89:ab:cd:ef +// 01-23-45-67-89-ab +// 01-23-45-67-89-ab-cd-ef +// 0123.4567.89ab +// 0123.4567.89ab.cdef +func ParseMAC(s string) (hw HardwareAddr, err os.Error) { + if len(s) < 14 { + goto error + } + + if s[2] == ':' || s[2] == '-' { + if (len(s)+1)%3 != 0 { + goto error + } + n := (len(s) + 1) / 3 + if n != 6 && n != 8 { + goto error + } + hw = make(HardwareAddr, n) + for x, i := 0, 0; i < n; i++ { + var ok bool + if hw[i], ok = xtoi2(s[x:], s[2]); !ok { + goto error + } + x += 3 + } + } else if s[4] == '.' { + if (len(s)+1)%5 != 0 { + goto error + } + n := 2 * (len(s) + 1) / 5 + if n != 6 && n != 8 { + goto error + } + hw = make(HardwareAddr, n) + for x, i := 0, 0; i < n; i += 2 { + var ok bool + if hw[i], ok = xtoi2(s[x:x+2], 0); !ok { + goto error + } + if hw[i+1], ok = xtoi2(s[x+2:], s[4]); !ok { + goto error + } + x += 5 + } + } else { + goto error + } + return hw, nil + +error: + return nil, os.NewError("invalid MAC address: " + s) +} + // Interface represents a mapping between network interface name // and index. It also represents network interface facility // information. diff --git a/libgo/go/net/interface_bsd.go b/libgo/go/net/interface_bsd.go index 2675f94..54fa5dd 100644 --- a/libgo/go/net/interface_bsd.go +++ b/libgo/go/net/interface_bsd.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd openbsd + // Network interface identification for BSD variants package net @@ -148,7 +150,6 @@ func newAddr(m *syscall.InterfaceAddrMessage) ([]Addr, os.Error) { } for _, s := range sas { - switch v := s.(type) { case *syscall.SockaddrInet4: ifa := &IPAddr{IP: IPv4(v.Addr[0], v.Addr[1], v.Addr[2], v.Addr[3])} diff --git a/libgo/go/net/interface_linux.go b/libgo/go/net/interface_linux.go index 3d2a0bb..36ae04ff 100644 --- a/libgo/go/net/interface_linux.go +++ b/libgo/go/net/interface_linux.go @@ -103,42 +103,29 @@ func linkFlags(rawFlags uint32) Flags { // for a specific interface. func interfaceAddrTable(ifindex int) ([]Addr, os.Error) { var ( - tab []byte - e int - err os.Error - ifat4 []Addr - ifat6 []Addr - msgs4 []syscall.NetlinkMessage - msgs6 []syscall.NetlinkMessage + tab []byte + e int + err os.Error + ifat []Addr + msgs []syscall.NetlinkMessage ) - tab, e = syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_INET) + tab, e = syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC) if e != 0 { return nil, os.NewSyscallError("netlink rib", e) } - msgs4, e = syscall.ParseNetlinkMessage(tab) - if e != 0 { - return nil, os.NewSyscallError("netlink message", e) - } - ifat4, err = addrTable(msgs4, ifindex) - if err != nil { - return nil, err - } - tab, e = syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_INET6) - if e != 0 { - return nil, os.NewSyscallError("netlink rib", e) - } - msgs6, e = syscall.ParseNetlinkMessage(tab) + msgs, e = syscall.ParseNetlinkMessage(tab) if e != 0 { return nil, os.NewSyscallError("netlink message", e) } - ifat6, err = addrTable(msgs6, ifindex) + + ifat, err = addrTable(msgs, ifindex) if err != nil { return nil, err } - return append(ifat4, ifat6...), nil + return ifat, nil } func addrTable(msgs []syscall.NetlinkMessage, ifindex int) ([]Addr, os.Error) { diff --git a/libgo/go/net/interface_stub.go b/libgo/go/net/interface_stub.go index 950de6c..282b38b 100644 --- a/libgo/go/net/interface_stub.go +++ b/libgo/go/net/interface_stub.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build plan9 + // Network interface identification package net diff --git a/libgo/go/net/interface_test.go b/libgo/go/net/interface_test.go index 0e4089a..c918f24 100644 --- a/libgo/go/net/interface_test.go +++ b/libgo/go/net/interface_test.go @@ -6,6 +6,9 @@ package net import ( "bytes" + "os" + "reflect" + "strings" "testing" ) @@ -71,3 +74,46 @@ func TestInterfaceAddrs(t *testing.T) { t.Logf("interface address %q\n", ifa.String()) } } + +var mactests = []struct { + in string + out HardwareAddr + err string +}{ + {"01:23:45:67:89:AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, + {"01-23-45-67-89-AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, + {"0123.4567.89AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, + {"ab:cd:ef:AB:CD:EF", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef}, ""}, + {"01.02.03.04.05.06", nil, "invalid MAC address"}, + {"01:02:03:04:05:06:", nil, "invalid MAC address"}, + {"x1:02:03:04:05:06", nil, "invalid MAC address"}, + {"01002:03:04:05:06", nil, "invalid MAC address"}, + {"01:02003:04:05:06", nil, "invalid MAC address"}, + {"01:02:03004:05:06", nil, "invalid MAC address"}, + {"01:02:03:04005:06", nil, "invalid MAC address"}, + {"01:02:03:04:05006", nil, "invalid MAC address"}, + {"01-02:03:04:05:06", nil, "invalid MAC address"}, + {"01:02-03-04-05-06", nil, "invalid MAC address"}, + {"0123:4567:89AF", nil, "invalid MAC address"}, + {"0123-4567-89AF", nil, "invalid MAC address"}, + {"01:23:45:67:89:AB:CD:EF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, + {"01-23-45-67-89-AB-CD-EF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, + {"0123.4567.89AB.CDEF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, +} + +func match(err os.Error, s string) bool { + if s == "" { + return err == nil + } + return err != nil && strings.Contains(err.String(), s) +} + +func TestParseMAC(t *testing.T) { + for _, tt := range mactests { + out, err := ParseMAC(tt.in) + if !reflect.DeepEqual(out, tt.out) || !match(err, tt.err) { + t.Errorf("ParseMAC(%q) = %v, %v, want %v, %v", tt.in, out, err, tt.out, + tt.err) + } + } +} diff --git a/libgo/go/net/ip.go b/libgo/go/net/ip.go index b0e2c42..61dc3be9 100644 --- a/libgo/go/net/ip.go +++ b/libgo/go/net/ip.go @@ -21,11 +21,8 @@ const ( ) // An IP is a single IP address, an array of bytes. -// Functions in this package accept either 4-byte (IP v4) -// or 16-byte (IP v6) arrays as input. Unless otherwise -// specified, functions in this package always return -// IP addresses in 16-byte form using the canonical -// embedding. +// Functions in this package accept either 4-byte (IPv4) +// or 16-byte (IPv6) arrays as input. // // Note that in this documentation, referring to an // IP address as an IPv4 address or an IPv6 address @@ -37,6 +34,12 @@ type IP []byte // An IP mask is an IP address. type IPMask []byte +// An IPNet represents an IP network. +type IPNet struct { + IP IP // network number + Mask IPMask // network mask +} + // IPv4 returns the IP address (in 16-byte form) of the // IPv4 address a.b.c.d. func IPv4(a, b, c, d byte) IP { @@ -51,20 +54,42 @@ func IPv4(a, b, c, d byte) IP { var v4InV6Prefix = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff} -// IPv4Mask returns the IP mask (in 16-byte form) of the +// IPv4Mask returns the IP mask (in 4-byte form) of the // IPv4 mask a.b.c.d. func IPv4Mask(a, b, c, d byte) IPMask { - p := make(IPMask, IPv6len) - for i := 0; i < 12; i++ { - p[i] = 0xff - } - p[12] = a - p[13] = b - p[14] = c - p[15] = d + p := make(IPMask, IPv4len) + p[0] = a + p[1] = b + p[2] = c + p[3] = d return p } +// CIDRMask returns an IPMask consisting of `ones' 1 bits +// followed by 0s up to a total length of `bits' bits. +// For a mask of this form, CIDRMask is the inverse of IPMask.Size. +func CIDRMask(ones, bits int) IPMask { + if bits != 8*IPv4len && bits != 8*IPv6len { + return nil + } + if ones < 0 || ones > bits { + return nil + } + l := bits / 8 + m := make(IPMask, l) + n := uint(ones) + for i := 0; i < l; i++ { + if n >= 8 { + m[i] = 0xff + n -= 8 + continue + } + m[i] = ^byte(0xff >> n) + n = 0 + } + return m +} + // Well-known IPv4 addresses var ( IPv4bcast = IPv4(255, 255, 255, 255) // broadcast @@ -213,13 +238,13 @@ func allFF(b []byte) bool { // Mask returns the result of masking the IP address ip with mask. func (ip IP) Mask(mask IPMask) IP { - n := len(ip) - if len(mask) == 16 && len(ip) == 4 && allFF(mask[:12]) { + if len(mask) == IPv6len && len(ip) == IPv4len && allFF(mask[:12]) { mask = mask[12:] } - if len(mask) == 4 && len(ip) == 16 && bytesEqual(ip[:12], v4InV6Prefix) { + if len(mask) == IPv4len && len(ip) == IPv6len && bytesEqual(ip[:12], v4InV6Prefix) { ip = ip[12:] } + n := len(ip) if n != len(mask) { return nil } @@ -230,40 +255,6 @@ func (ip IP) Mask(mask IPMask) IP { return out } -// Convert i to decimal string. -func itod(i uint) string { - if i == 0 { - return "0" - } - - // Assemble decimal in reverse order. - var b [32]byte - bp := len(b) - for ; i > 0; i /= 10 { - bp-- - b[bp] = byte(i%10) + '0' - } - - return string(b[bp:]) -} - -// Convert i to hexadecimal string. -func itox(i uint) string { - if i == 0 { - return "0" - } - - // Assemble hexadecimal in reverse order. - var b [32]byte - bp := len(b) - for ; i > 0; i /= 16 { - bp-- - b[bp] = "0123456789abcdef"[byte(i%16)] - } - - return string(b[bp:]) -} - // String returns the string form of the IP address ip. // If the address is an IPv4 address, the string representation // is dotted decimal ("74.125.19.99"). Otherwise the representation @@ -272,11 +263,11 @@ func (ip IP) String() string { p := ip if len(ip) == 0 { - return "" + return "<nil>" } // If IPv4, use dotted notation. - if p4 := p.To4(); len(p4) == 4 { + if p4 := p.To4(); len(p4) == IPv4len { return itod(uint(p4[0])) + "." + itod(uint(p4[1])) + "." + itod(uint(p4[2])) + "." + @@ -289,9 +280,9 @@ func (ip IP) String() string { // Find longest run of zeros. e0 := -1 e1 := -1 - for i := 0; i < 16; i += 2 { + for i := 0; i < IPv6len; i += 2 { j := i - for j < 16 && p[j] == 0 && p[j+1] == 0 { + for j < IPv6len && p[j] == 0 && p[j+1] == 0 { j += 2 } if j > i && j-i > e1-e0 { @@ -307,17 +298,17 @@ func (ip IP) String() string { // Print with possible :: in place of run of zeros var s string - for i := 0; i < 16; i += 2 { + for i := 0; i < IPv6len; i += 2 { if i == e0 { s += "::" i = e1 - if i >= 16 { + if i >= IPv6len { break } } else if i > 0 { s += ":" } - s += itox((uint(p[i]) << 8) | uint(p[i+1])) + s += itox((uint(p[i])<<8)|uint(p[i+1]), 1) } return s } @@ -329,10 +320,10 @@ func (ip IP) Equal(x IP) bool { if len(ip) == len(x) { return bytesEqual(ip, x) } - if len(ip) == 4 && len(x) == 16 { + if len(ip) == IPv4len && len(x) == IPv6len { return bytesEqual(x[0:12], v4InV6Prefix) && bytesEqual(ip, x[12:]) } - if len(ip) == 16 && len(x) == 4 { + if len(ip) == IPv6len && len(x) == IPv4len { return bytesEqual(ip[0:12], v4InV6Prefix) && bytesEqual(ip[12:], x) } return false @@ -379,25 +370,86 @@ func simpleMaskLength(mask IPMask) int { return n } -// String returns the string representation of mask. -// If the mask is in the canonical form--ones followed by zeros--the -// string representation is just the decimal number of ones. -// If the mask is in a non-canonical form, it is formatted -// as an IP address. -func (mask IPMask) String() string { - switch len(mask) { - case 4: - n := simpleMaskLength(mask) - if n >= 0 { - return itod(uint(n + (IPv6len-IPv4len)*8)) +// Size returns the number of leading ones and total bits in the mask. +// If the mask is not in the canonical form--ones followed by zeros--then +// Size returns 0, 0. +func (m IPMask) Size() (ones, bits int) { + ones, bits = simpleMaskLength(m), len(m)*8 + if ones == -1 { + return 0, 0 + } + return +} + +// String returns the hexadecimal form of m, with no punctuation. +func (m IPMask) String() string { + s := "" + for _, b := range m { + s += itox(uint(b), 2) + } + if len(s) == 0 { + return "<nil>" + } + return s +} + +func networkNumberAndMask(n *IPNet) (ip IP, m IPMask) { + if ip = n.IP.To4(); ip == nil { + ip = n.IP + if len(ip) != IPv6len { + return nil, nil } - case 16: - n := simpleMaskLength(mask) - if n >= 12*8 { - return itod(uint(n - 12*8)) + } + m = n.Mask + switch len(m) { + case IPv4len: + if len(ip) != IPv4len { + return nil, nil + } + case IPv6len: + if len(ip) == IPv4len { + m = m[12:] + } + default: + return nil, nil + } + return +} + +// Contains reports whether the network includes ip. +func (n *IPNet) Contains(ip IP) bool { + nn, m := networkNumberAndMask(n) + if x := ip.To4(); x != nil { + ip = x + } + l := len(ip) + if l != len(nn) { + return false + } + for i := 0; i < l; i++ { + if nn[i]&m[i] != ip[i]&m[i] { + return false } } - return IP(mask).String() + return true +} + +// String returns the CIDR notation of n like "192.168.100.1/24" +// or "2001:DB8::/48" as defined in RFC 4632 and RFC 4291. +// If the mask is not in the canonical form, it returns the +// string which consists of an IP address, followed by a slash +// character and a mask expressed as hexadecimal form with no +// punctuation like "192.168.100.1/c000ff00". +func (n *IPNet) String() string { + nn, m := networkNumberAndMask(n) + if nn == nil || m == nil { + return "<nil>" + } + l := simpleMaskLength(m) + if l == -1 { + return nn.String() + "/" + m.String() + } + return nn.String() + "/" + itod(uint(l)) } // Parse IPv4 address (d.d.d.d). @@ -440,7 +492,7 @@ func parseIPv4(s string) IP { // * The last 32 bits can be in IPv4 form. // Thus, ::ffff:1.2.3.4 is the IPv4 address 1.2.3.4. func parseIPv6(s string) IP { - p := make(IP, 16) + p := make(IP, IPv6len) ellipsis := -1 // position of ellipsis in p i := 0 // index in string s @@ -482,7 +534,7 @@ func parseIPv6(s string) IP { p[j+2] = p4[14] p[j+3] = p4[15] i = len(s) - j += 4 + j += IPv4len break } @@ -569,46 +621,28 @@ func ParseIP(s string) IP { } // ParseCIDR parses s as a CIDR notation IP address and mask, -// like "192.168.100.1/24", "2001:DB8::/48", as defined in +// like "192.168.100.1/24" or "2001:DB8::/48", as defined in // RFC 4632 and RFC 4291. -func ParseCIDR(s string) (ip IP, mask IPMask, err os.Error) { +// +// It returns the IP address and the network implied by the IP +// and mask. For example, ParseCIDR("192.168.100.1/16") returns +// the IP address 192.168.100.1 and the network 192.168.0.0/16. +func ParseCIDR(s string) (IP, *IPNet, os.Error) { i := byteIndex(s, '/') if i < 0 { return nil, nil, &ParseError{"CIDR address", s} } ipstr, maskstr := s[:i], s[i+1:] - iplen := 4 - ip = parseIPv4(ipstr) + iplen := IPv4len + ip := parseIPv4(ipstr) if ip == nil { - iplen = 16 + iplen = IPv6len ip = parseIPv6(ipstr) } - nn, i, ok := dtoi(maskstr, 0) - if ip == nil || !ok || i != len(maskstr) || nn < 0 || nn > 8*iplen { + n, i, ok := dtoi(maskstr, 0) + if ip == nil || !ok || i != len(maskstr) || n < 0 || n > 8*iplen { return nil, nil, &ParseError{"CIDR address", s} } - n := uint(nn) - if iplen == 4 { - v4mask := ^uint32(0xffffffff >> n) - mask = IPv4Mask(byte(v4mask>>24), byte(v4mask>>16), byte(v4mask>>8), byte(v4mask)) - } else { - mask = make(IPMask, 16) - for i := 0; i < 16; i++ { - if n >= 8 { - mask[i] = 0xff - n -= 8 - continue - } - mask[i] = ^byte(0xff >> n) - n = 0 - - } - } - // address must not have any bits not in mask - for i := range ip { - if ip[i]&^mask[i] != 0 { - return nil, nil, &ParseError{"CIDR address", s} - } - } - return ip, mask, nil + m := CIDRMask(n, 8*iplen) + return ip, &IPNet{ip.Mask(m), m}, nil } diff --git a/libgo/go/net/ip_test.go b/libgo/go/net/ip_test.go index b189b10..07e627a 100644 --- a/libgo/go/net/ip_test.go +++ b/libgo/go/net/ip_test.go @@ -34,12 +34,13 @@ var parseiptests = []struct { {"::ffff:127.0.0.1", IPv4(127, 0, 0, 1)}, {"2001:4860:0:2001::68", IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68}}, {"::ffff:4a7d:1363", IPv4(74, 125, 19, 99)}, + {"", nil}, } func TestParseIP(t *testing.T) { for _, tt := range parseiptests { if out := ParseIP(tt.in); !isEqual(out, tt.out) { - t.Errorf("ParseIP(%#q) = %v, want %v", tt.in, out, tt.out) + t.Errorf("ParseIP(%q) = %v, want %v", tt.in, out, tt.out) } } } @@ -49,60 +50,213 @@ var ipstringtests = []struct { out string }{ // cf. RFC 5952 (A Recommendation for IPv6 Address Text Representation) - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, - "2001:db8::123:12:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, - "2001:db8::1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, - "2001:db8:0:1:0:1:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, - "2001:db8:1:0:1:0:1:0"}, - {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, - "2001::1:0:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, - "2001:db8:0:0:1::"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, - "2001:db8::1:0:0:1"}, - {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, - "2001:db8::a:b:c:d"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, "2001:db8::123:12:1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, "2001:db8:0:1:0:1:0:1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, "2001:db8:1:0:1:0:1:0"}, + {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001::1:0:0:1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, "2001:db8:0:0:1::"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1:0:0:1"}, + {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, "2001:db8::a:b:c:d"}, + {nil, "<nil>"}, } func TestIPString(t *testing.T) { for _, tt := range ipstringtests { if out := tt.in.String(); out != tt.out { - t.Errorf("IP.String(%v) = %#q, want %#q", tt.in, out, tt.out) + t.Errorf("IP.String(%v) = %q, want %q", tt.in, out, tt.out) } } } -var parsecidrtests = []struct { - in string - ip IP +var ipmasktests = []struct { + in IP mask IPMask - err os.Error + out IP +}{ + {IPv4(192, 168, 1, 127), IPv4Mask(255, 255, 255, 128), IPv4(192, 168, 1, 0)}, + {IPv4(192, 168, 1, 127), IPMask(ParseIP("255.255.255.192")), IPv4(192, 168, 1, 64)}, + {IPv4(192, 168, 1, 127), IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffe0")), IPv4(192, 168, 1, 96)}, + {IPv4(192, 168, 1, 127), IPv4Mask(255, 0, 255, 0), IPv4(192, 0, 1, 0)}, + {ParseIP("2001:db8::1"), IPMask(ParseIP("ffff:ff80::")), ParseIP("2001:d80::")}, + {ParseIP("2001:db8::1"), IPMask(ParseIP("f0f0:0f0f::")), ParseIP("2000:d08::")}, +} + +func TestIPMask(t *testing.T) { + for _, tt := range ipmasktests { + if out := tt.in.Mask(tt.mask); out == nil || !tt.out.Equal(out) { + t.Errorf("IP(%v).Mask(%v) = %v, want %v", tt.in, tt.mask, out, tt.out) + } + } +} + +var ipmaskstringtests = []struct { + in IPMask + out string +}{ + {IPv4Mask(255, 255, 255, 240), "fffffff0"}, + {IPv4Mask(255, 0, 128, 0), "ff008000"}, + {IPMask(ParseIP("ffff:ff80::")), "ffffff80000000000000000000000000"}, + {IPMask(ParseIP("ef00:ff80::cafe:0")), "ef00ff800000000000000000cafe0000"}, + {nil, "<nil>"}, +} + +func TestIPMaskString(t *testing.T) { + for _, tt := range ipmaskstringtests { + if out := tt.in.String(); out != tt.out { + t.Errorf("IPMask.String(%v) = %q, want %q", tt.in, out, tt.out) + } + } +} + +var parsecidrtests = []struct { + in string + ip IP + net *IPNet + err os.Error }{ - {"135.104.0.0/32", IPv4(135, 104, 0, 0), IPv4Mask(255, 255, 255, 255), nil}, - {"0.0.0.0/24", IPv4(0, 0, 0, 0), IPv4Mask(255, 255, 255, 0), nil}, - {"135.104.0.0/24", IPv4(135, 104, 0, 0), IPv4Mask(255, 255, 255, 0), nil}, - {"135.104.0.1/32", IPv4(135, 104, 0, 1), IPv4Mask(255, 255, 255, 255), nil}, - {"135.104.0.1/24", nil, nil, &ParseError{"CIDR address", "135.104.0.1/24"}}, - {"::1/128", ParseIP("::1"), IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")), nil}, - {"abcd:2345::/127", ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe")), nil}, - {"abcd:2345::/65", ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:ffff:8000::")), nil}, - {"abcd:2345::/64", ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:ffff::")), nil}, - {"abcd:2345::/63", ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:fffe::")), nil}, - {"abcd:2345::/33", ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:8000::")), nil}, - {"abcd:2345::/32", ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff::")), nil}, - {"abcd:2344::/31", ParseIP("abcd:2344::"), IPMask(ParseIP("ffff:fffe::")), nil}, - {"abcd:2300::/24", ParseIP("abcd:2300::"), IPMask(ParseIP("ffff:ff00::")), nil}, - {"abcd:2345::/24", nil, nil, &ParseError{"CIDR address", "abcd:2345::/24"}}, - {"2001:DB8::/48", ParseIP("2001:DB8::"), IPMask(ParseIP("ffff:ffff:ffff::")), nil}, + {"135.104.0.0/32", IPv4(135, 104, 0, 0), &IPNet{IPv4(135, 104, 0, 0), IPv4Mask(255, 255, 255, 255)}, nil}, + {"0.0.0.0/24", IPv4(0, 0, 0, 0), &IPNet{IPv4(0, 0, 0, 0), IPv4Mask(255, 255, 255, 0)}, nil}, + {"135.104.0.0/24", IPv4(135, 104, 0, 0), &IPNet{IPv4(135, 104, 0, 0), IPv4Mask(255, 255, 255, 0)}, nil}, + {"135.104.0.1/32", IPv4(135, 104, 0, 1), &IPNet{IPv4(135, 104, 0, 1), IPv4Mask(255, 255, 255, 255)}, nil}, + {"135.104.0.1/24", IPv4(135, 104, 0, 1), &IPNet{IPv4(135, 104, 0, 0), IPv4Mask(255, 255, 255, 0)}, nil}, + {"::1/128", ParseIP("::1"), &IPNet{ParseIP("::1"), IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"))}, nil}, + {"abcd:2345::/127", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"))}, nil}, + {"abcd:2345::/65", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:ffff:8000::"))}, nil}, + {"abcd:2345::/64", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:ffff::"))}, nil}, + {"abcd:2345::/63", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:ffff:fffe::"))}, nil}, + {"abcd:2345::/33", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff:8000::"))}, nil}, + {"abcd:2345::/32", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2345::"), IPMask(ParseIP("ffff:ffff::"))}, nil}, + {"abcd:2344::/31", ParseIP("abcd:2344::"), &IPNet{ParseIP("abcd:2344::"), IPMask(ParseIP("ffff:fffe::"))}, nil}, + {"abcd:2300::/24", ParseIP("abcd:2300::"), &IPNet{ParseIP("abcd:2300::"), IPMask(ParseIP("ffff:ff00::"))}, nil}, + {"abcd:2345::/24", ParseIP("abcd:2345::"), &IPNet{ParseIP("abcd:2300::"), IPMask(ParseIP("ffff:ff00::"))}, nil}, + {"2001:DB8::/48", ParseIP("2001:DB8::"), &IPNet{ParseIP("2001:DB8::"), IPMask(ParseIP("ffff:ffff:ffff::"))}, nil}, + {"2001:DB8::1/48", ParseIP("2001:DB8::1"), &IPNet{ParseIP("2001:DB8::"), IPMask(ParseIP("ffff:ffff:ffff::"))}, nil}, + {"192.168.1.1/255.255.255.0", nil, nil, &ParseError{"CIDR address", "192.168.1.1/255.255.255.0"}}, + {"192.168.1.1/35", nil, nil, &ParseError{"CIDR address", "192.168.1.1/35"}}, + {"2001:db8::1/-1", nil, nil, &ParseError{"CIDR address", "2001:db8::1/-1"}}, + {"", nil, nil, &ParseError{"CIDR address", ""}}, } func TestParseCIDR(t *testing.T) { for _, tt := range parsecidrtests { - if ip, mask, err := ParseCIDR(tt.in); !isEqual(ip, tt.ip) || !isEqual(mask, tt.mask) || !reflect.DeepEqual(err, tt.err) { - t.Errorf("ParseCIDR(%q) = %v, %v, %v; want %v, %v, %v", tt.in, ip, mask, err, tt.ip, tt.mask, tt.err) + ip, net, err := ParseCIDR(tt.in) + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("ParseCIDR(%q) = %v, %v; want %v, %v", tt.in, ip, net, tt.ip, tt.net) + } + if err == nil && (!tt.ip.Equal(ip) || !tt.net.IP.Equal(net.IP) || !isEqual(net.Mask, tt.net.Mask)) { + t.Errorf("ParseCIDR(%q) = %v, {%v, %v}; want %v {%v, %v}", tt.in, ip, net.IP, net.Mask, tt.ip, tt.net.IP, tt.net.Mask) + } + } +} + +var ipnetcontainstests = []struct { + ip IP + net *IPNet + ok bool +}{ + {IPv4(172, 16, 1, 1), &IPNet{IPv4(172, 16, 0, 0), CIDRMask(12, 32)}, true}, + {IPv4(172, 24, 0, 1), &IPNet{IPv4(172, 16, 0, 0), CIDRMask(13, 32)}, false}, + {IPv4(192, 168, 0, 3), &IPNet{IPv4(192, 168, 0, 0), IPv4Mask(0, 0, 255, 252)}, true}, + {IPv4(192, 168, 0, 4), &IPNet{IPv4(192, 168, 0, 0), IPv4Mask(0, 255, 0, 252)}, false}, + {ParseIP("2001:db8:1:2::1"), &IPNet{ParseIP("2001:db8:1::"), CIDRMask(47, 128)}, true}, + {ParseIP("2001:db8:1:2::1"), &IPNet{ParseIP("2001:db8:2::"), CIDRMask(47, 128)}, false}, + {ParseIP("2001:db8:1:2::1"), &IPNet{ParseIP("2001:db8:1::"), IPMask(ParseIP("ffff:0:ffff::"))}, true}, + {ParseIP("2001:db8:1:2::1"), &IPNet{ParseIP("2001:db8:1::"), IPMask(ParseIP("0:0:0:ffff::"))}, false}, +} + +func TestIPNetContains(t *testing.T) { + for _, tt := range ipnetcontainstests { + if ok := tt.net.Contains(tt.ip); ok != tt.ok { + t.Errorf("IPNet(%v).Contains(%v) = %v, want %v", tt.net, tt.ip, ok, tt.ok) + } + } +} + +var ipnetstringtests = []struct { + in *IPNet + out string +}{ + {&IPNet{IPv4(192, 168, 1, 0), CIDRMask(26, 32)}, "192.168.1.0/26"}, + {&IPNet{IPv4(192, 168, 1, 0), IPv4Mask(255, 0, 255, 0)}, "192.168.1.0/ff00ff00"}, + {&IPNet{ParseIP("2001:db8::"), CIDRMask(55, 128)}, "2001:db8::/55"}, + {&IPNet{ParseIP("2001:db8::"), IPMask(ParseIP("8000:f123:0:cafe::"))}, "2001:db8::/8000f1230000cafe0000000000000000"}, +} + +func TestIPNetString(t *testing.T) { + for _, tt := range ipnetstringtests { + if out := tt.in.String(); out != tt.out { + t.Errorf("IPNet.String(%v) = %q, want %q", tt.in, out, tt.out) + } + } +} + +var cidrmasktests = []struct { + ones int + bits int + out IPMask +}{ + {0, 32, IPv4Mask(0, 0, 0, 0)}, + {12, 32, IPv4Mask(255, 240, 0, 0)}, + {24, 32, IPv4Mask(255, 255, 255, 0)}, + {32, 32, IPv4Mask(255, 255, 255, 255)}, + {0, 128, IPMask{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {4, 128, IPMask{0xf0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {48, 128, IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {128, 128, IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {33, 32, nil}, + {32, 33, nil}, + {-1, 128, nil}, + {128, -1, nil}, +} + +func TestCIDRMask(t *testing.T) { + for _, tt := range cidrmasktests { + if out := CIDRMask(tt.ones, tt.bits); !isEqual(out, tt.out) { + t.Errorf("CIDRMask(%v, %v) = %v, want %v", tt.ones, tt.bits, out, tt.out) + } + } +} + +var ( + v4addr = IP{192, 168, 0, 1} + v4mappedv6addr = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 0, 1} + v6addr = IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1} + v4mask = IPMask{255, 255, 255, 0} + v4mappedv6mask = IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 255, 255, 255, 0} + v6mask = IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0} + badaddr = IP{192, 168, 0} + badmask = IPMask{255, 255, 0} + v4maskzero = IPMask{0, 0, 0, 0} +) + +var networknumberandmasktests = []struct { + in IPNet + out IPNet +}{ + {IPNet{v4addr, v4mask}, IPNet{v4addr, v4mask}}, + {IPNet{v4addr, v4mappedv6mask}, IPNet{v4addr, v4mask}}, + {IPNet{v4mappedv6addr, v4mappedv6mask}, IPNet{v4addr, v4mask}}, + {IPNet{v4mappedv6addr, v6mask}, IPNet{v4addr, v4maskzero}}, + {IPNet{v4addr, v6mask}, IPNet{v4addr, v4maskzero}}, + {IPNet{v6addr, v6mask}, IPNet{v6addr, v6mask}}, + {IPNet{v6addr, v4mappedv6mask}, IPNet{v6addr, v4mappedv6mask}}, + {in: IPNet{v6addr, v4mask}}, + {in: IPNet{v4addr, badmask}}, + {in: IPNet{v4mappedv6addr, badmask}}, + {in: IPNet{v6addr, badmask}}, + {in: IPNet{badaddr, v4mask}}, + {in: IPNet{badaddr, v4mappedv6mask}}, + {in: IPNet{badaddr, v6mask}}, + {in: IPNet{badaddr, badmask}}, +} + +func TestNetworkNumberAndMask(t *testing.T) { + for _, tt := range networknumberandmasktests { + ip, m := networkNumberAndMask(&tt.in) + out := &IPNet{ip, m} + if !reflect.DeepEqual(&tt.out, out) { + t.Errorf("networkNumberAndMask(%v) = %v; want %v", tt.in, out, &tt.out) } } } @@ -158,10 +312,10 @@ var ipaftests = []struct { func TestIPAddrFamily(t *testing.T) { for _, tt := range ipaftests { if af := tt.in.To4() != nil; af != tt.af4 { - t.Errorf("verifying IPv4 address family for %#q = %v, want %v", tt.in, af, tt.af4) + t.Errorf("verifying IPv4 address family for %q = %v, want %v", tt.in, af, tt.af4) } if af := len(tt.in) == IPv6len && tt.in.To4() == nil; af != tt.af6 { - t.Errorf("verifying IPv6 address family for %#q = %v, want %v", tt.in, af, tt.af6) + t.Errorf("verifying IPv6 address family for %q = %v, want %v", tt.in, af, tt.af6) } } } @@ -209,7 +363,7 @@ func name(f interface{}) string { func TestIPAddrScope(t *testing.T) { for _, tt := range ipscopetests { if ok := tt.scope(tt.in); ok != tt.ok { - t.Errorf("%s(%#q) = %v, want %v", name(tt.scope), tt.in, ok, tt.ok) + t.Errorf("%s(%q) = %v, want %v", name(tt.scope), tt.in, ok, tt.ok) } } } diff --git a/libgo/go/net/iprawsock_posix.go b/libgo/go/net/iprawsock_posix.go index 4e11518..dafbdab 100644 --- a/libgo/go/net/iprawsock_posix.go +++ b/libgo/go/net/iprawsock_posix.go @@ -2,18 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + // (Raw) IP sockets package net import ( "os" - "sync" "syscall" ) -var onceReadProtocols sync.Once - func sockaddrToIP(sa syscall.Sockaddr) Addr { switch sa := sa.(type) { case *syscall.SockaddrInet4: @@ -25,7 +24,7 @@ func sockaddrToIP(sa syscall.Sockaddr) Addr { } func (a *IPAddr) family() int { - if a == nil || len(a.IP) <= 4 { + if a == nil || len(a.IP) <= IPv4len { return syscall.AF_INET } if a.IP.To4() != nil { @@ -158,7 +157,7 @@ func (c *IPConn) ReadFromIP(b []byte) (n int, addr *IPAddr, err os.Error) { switch sa := sa.(type) { case *syscall.SockaddrInet4: addr = &IPAddr{sa.Addr[0:]} - if len(b) >= 4 { // discard ipv4 header + if len(b) >= IPv4len { // discard ipv4 header hsize := (int(b[0]) & 0xf) * 4 copy(b, b[hsize:]) n -= hsize @@ -207,33 +206,7 @@ func (c *IPConn) WriteTo(b []byte, addr Addr) (n int, err os.Error) { return c.WriteToIP(b, a) } -var protocols map[string]int - -func readProtocols() { - protocols = make(map[string]int) - if file, err := open("/etc/protocols"); err == nil { - for line, ok := file.readLine(); ok; line, ok = file.readLine() { - // tcp 6 TCP # transmission control protocol - if i := byteIndex(line, '#'); i >= 0 { - line = line[0:i] - } - f := getFields(line) - if len(f) < 2 { - continue - } - if proto, _, ok := dtoi(f[1], 0); ok { - protocols[f[0]] = proto - for _, alias := range f[2:] { - protocols[alias] = proto - } - } - } - file.close() - } -} - func splitNetProto(netProto string) (net string, proto int, err os.Error) { - onceReadProtocols.Do(readProtocols) i := last(netProto, ':') if i < 0 { // no colon return "", 0, os.NewError("no IP protocol specified") @@ -242,13 +215,12 @@ func splitNetProto(netProto string) (net string, proto int, err os.Error) { protostr := netProto[i+1:] proto, i, ok := dtoi(protostr, 0) if !ok || i != len(protostr) { - // lookup by name - proto, ok = protocols[protostr] - if ok { - return + proto, err = lookupProtocol(protostr) + if err != nil { + return "", 0, err } } - return + return net, proto, nil } // DialIP connects to the remote address raddr on the network net, @@ -303,3 +275,8 @@ func (c *IPConn) BindToDevice(device string) os.Error { defer c.fd.decref() return os.NewSyscallError("setsockopt", syscall.BindToDevice(c.fd.sysfd, device)) } + +// File returns a copy of the underlying os.File, set to blocking mode. +// It is the caller's responsibility to close f when finished. +// Closing c does not affect f, and closing f does not affect c. +func (c *IPConn) File() (f *os.File, err os.Error) { return c.fd.dup() } diff --git a/libgo/go/net/ipsock_posix.go b/libgo/go/net/ipsock_posix.go index 0c522fb..049df9e 100644 --- a/libgo/go/net/ipsock_posix.go +++ b/libgo/go/net/ipsock_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + package net import ( diff --git a/libgo/go/net/lookup_plan9.go b/libgo/go/net/lookup_plan9.go index 37d6b8e..d779f4a 100644 --- a/libgo/go/net/lookup_plan9.go +++ b/libgo/go/net/lookup_plan9.go @@ -157,12 +157,21 @@ func LookupCNAME(name string) (cname string, err os.Error) { } // LookupSRV tries to resolve an SRV query of the given service, -// protocol, and domain name, as specified in RFC 2782. In most cases -// the proto argument can be the same as the corresponding -// Addr.Network(). The returned records are sorted by priority -// and randomized by weight within a priority. +// protocol, and domain name. The proto is "tcp" or "udp". +// The returned records are sorted by priority and randomized +// by weight within a priority. +// +// LookupSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _service._proto.name. To accommodate services +// publishing SRV records under non-standard names, if both service +// and proto are empty strings, LookupSRV looks up name directly. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err os.Error) { - target := "_" + service + "._" + proto + "." + name + var target string + if service == "" && proto == "" { + target = name + } else { + target = "_" + service + "._" + proto + "." + name + } lines, err := queryDNS(target, "srv") if err != nil { return @@ -204,6 +213,11 @@ func LookupMX(name string) (mx []*MX, err os.Error) { return } +// LookupTXT returns the DNS TXT records for the given domain name. +func LookupTXT(name string) (txt []string, err os.Error) { + return nil, os.NewError("net.LookupTXT is not implemented on Plan 9") +} + // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. func LookupAddr(addr string) (name []string, err os.Error) { diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index 995ab03..c0fcd26 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -26,6 +26,15 @@ func TestGoogleSRV(t *testing.T) { if len(addrs) == 0 { t.Errorf("no results") } + + // Non-standard back door. + _, addrs, err = LookupSRV("", "", "_xmpp-server._tcp.google.com") + if err != nil { + t.Errorf("back door failed: %s", err) + } + if len(addrs) == 0 { + t.Errorf("back door no results") + } } func TestGmailMX(t *testing.T) { @@ -42,6 +51,24 @@ func TestGmailMX(t *testing.T) { } } +func TestGmailTXT(t *testing.T) { + if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + t.Logf("LookupTXT is not implemented on Windows or Plan 9") + return + } + if testing.Short() || avoidMacFirewall { + t.Logf("skipping test to avoid external network") + return + } + txt, err := LookupTXT("gmail.com") + if err != nil { + t.Errorf("failed: %s", err) + } + if len(txt) == 0 || len(txt[0]) == 0 { + t.Errorf("no results") + } +} + func TestGoogleDNSAddr(t *testing.T) { if testing.Short() || avoidMacFirewall { t.Logf("skipping test to avoid external network") diff --git a/libgo/go/net/lookup_unix.go b/libgo/go/net/lookup_unix.go index 8f5e66212..6e79295 100644 --- a/libgo/go/net/lookup_unix.go +++ b/libgo/go/net/lookup_unix.go @@ -2,12 +2,56 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package net import ( "os" + "sync" +) + +var ( + protocols map[string]int + onceReadProtocols sync.Once ) +// readProtocols loads contents of /etc/protocols into protocols map +// for quick access. +func readProtocols() { + protocols = make(map[string]int) + if file, err := open("/etc/protocols"); err == nil { + for line, ok := file.readLine(); ok; line, ok = file.readLine() { + // tcp 6 TCP # transmission control protocol + if i := byteIndex(line, '#'); i >= 0 { + line = line[0:i] + } + f := getFields(line) + if len(f) < 2 { + continue + } + if proto, _, ok := dtoi(f[1], 0); ok { + protocols[f[0]] = proto + for _, alias := range f[2:] { + protocols[alias] = proto + } + } + } + file.close() + } +} + +// lookupProtocol looks up IP protocol name in /etc/protocols and +// returns correspondent protocol number. +func lookupProtocol(name string) (proto int, err os.Error) { + onceReadProtocols.Do(readProtocols) + proto, found := protocols[name] + if !found { + return 0, os.NewError("unknown IP protocol specified: " + name) + } + return +} + // LookupHost looks up the given host using the local resolver. // It returns an array of that host's addresses. func LookupHost(host string) (addrs []string, err os.Error) { @@ -50,12 +94,21 @@ func LookupCNAME(name string) (cname string, err os.Error) { } // LookupSRV tries to resolve an SRV query of the given service, -// protocol, and domain name, as specified in RFC 2782. In most cases -// the proto argument can be the same as the corresponding -// Addr.Network(). The returned records are sorted by priority -// and randomized by weight within a priority. +// protocol, and domain name. The proto is "tcp" or "udp". +// The returned records are sorted by priority and randomized +// by weight within a priority. +// +// LookupSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _service._proto.name. To accommodate services +// publishing SRV records under non-standard names, if both service +// and proto are empty strings, LookupSRV looks up name directly. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err os.Error) { - target := "_" + service + "._" + proto + "." + name + var target string + if service == "" && proto == "" { + target = name + } else { + target = "_" + service + "._" + proto + "." + name + } var records []dnsRR cname, records, err = lookup(target, dnsTypeSRV) if err != nil { @@ -72,19 +125,32 @@ func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err os. // LookupMX returns the DNS MX records for the given domain name sorted by preference. func LookupMX(name string) (mx []*MX, err os.Error) { - _, rr, err := lookup(name, dnsTypeMX) + _, records, err := lookup(name, dnsTypeMX) if err != nil { return } - mx = make([]*MX, len(rr)) - for i := range rr { - r := rr[i].(*dnsRR_MX) + mx = make([]*MX, len(records)) + for i, rr := range records { + r := rr.(*dnsRR_MX) mx[i] = &MX{r.Mx, r.Pref} } byPref(mx).sort() return } +// LookupTXT returns the DNS TXT records for the given domain name. +func LookupTXT(name string) (txt []string, err os.Error) { + _, records, err := lookup(name, dnsTypeTXT) + if err != nil { + return + } + txt = make([]string, len(records)) + for i, r := range records { + txt[i] = r.(*dnsRR_TXT).Txt + } + return +} + // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. func LookupAddr(addr string) (name []string, err os.Error) { diff --git a/libgo/go/net/lookup_windows.go b/libgo/go/net/lookup_windows.go index fa3ad7c..ea939f8 100644 --- a/libgo/go/net/lookup_windows.go +++ b/libgo/go/net/lookup_windows.go @@ -11,8 +11,22 @@ import ( "sync" ) -var hostentLock sync.Mutex -var serventLock sync.Mutex +var ( + protoentLock sync.Mutex + hostentLock sync.Mutex + serventLock sync.Mutex +) + +// lookupProtocol looks up IP protocol name and returns correspondent protocol number. +func lookupProtocol(name string) (proto int, err os.Error) { + protoentLock.Lock() + defer protoentLock.Unlock() + p, e := syscall.GetProtoByName(name) + if e != 0 { + return 0, os.NewSyscallError("GetProtoByName", e) + } + return int(p.Proto), nil +} func LookupHost(name string) (addrs []string, err os.Error) { ips, err := LookupIP(name) @@ -77,9 +91,23 @@ func LookupCNAME(name string) (cname string, err os.Error) { return } +// LookupSRV tries to resolve an SRV query of the given service, +// protocol, and domain name. The proto is "tcp" or "udp". +// The returned records are sorted by priority and randomized +// by weight within a priority. +// +// LookupSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _service._proto.name. To accommodate services +// publishing SRV records under non-standard names, if both service +// and proto are empty strings, LookupSRV looks up name directly. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err os.Error) { + var target string + if service == "" && proto == "" { + target = name + } else { + target = "_" + service + "._" + proto + "." + name + } var r *syscall.DNSRecord - target := "_" + service + "._" + proto + "." + name e := syscall.DnsQuery(target, syscall.DNS_TYPE_SRV, 0, nil, &r, nil) if int(e) != 0 { return "", nil, os.NewSyscallError("LookupSRV", int(e)) @@ -110,6 +138,10 @@ func LookupMX(name string) (mx []*MX, err os.Error) { return mx, nil } +func LookupTXT(name string) (txt []string, err os.Error) { + return nil, os.NewError("net.LookupTXT is not implemented on Windows") +} + func LookupAddr(addr string) (name []string, err os.Error) { arpa, err := reverseaddr(addr) if err != nil { diff --git a/libgo/go/net/multicast_test.go b/libgo/go/net/multicast_test.go index be6dbf2..a66250c 100644 --- a/libgo/go/net/multicast_test.go +++ b/libgo/go/net/multicast_test.go @@ -6,13 +6,33 @@ package net import ( "flag" + "os" "runtime" "testing" ) var multicast = flag.Bool("multicast", false, "enable multicast tests") -func TestMulticastJoinAndLeave(t *testing.T) { +var joinAndLeaveGroupUDPTests = []struct { + net string + laddr IP + gaddr IP + flags Flags + ipv6 bool +}{ + // cf. RFC 4727: Experimental Values in IPv4, IPv6, ICMPv4, ICMPv6, UDP, and TCP Headers + {"udp", IPv4zero, IPv4(224, 0, 0, 254), (FlagUp | FlagLoopback), false}, + {"udp4", IPv4zero, IPv4(224, 0, 0, 254), (FlagUp | FlagLoopback), false}, + {"udp", IPv6unspecified, ParseIP("ff0e::114"), (FlagUp | FlagLoopback), true}, + {"udp6", IPv6unspecified, ParseIP("ff01::114"), (FlagUp | FlagLoopback), true}, + {"udp6", IPv6unspecified, ParseIP("ff02::114"), (FlagUp | FlagLoopback), true}, + {"udp6", IPv6unspecified, ParseIP("ff04::114"), (FlagUp | FlagLoopback), true}, + {"udp6", IPv6unspecified, ParseIP("ff05::114"), (FlagUp | FlagLoopback), true}, + {"udp6", IPv6unspecified, ParseIP("ff08::114"), (FlagUp | FlagLoopback), true}, + {"udp6", IPv6unspecified, ParseIP("ff0e::114"), (FlagUp | FlagLoopback), true}, +} + +func TestJoinAndLeaveGroupUDP(t *testing.T) { if runtime.GOOS == "windows" { return } @@ -21,53 +41,51 @@ func TestMulticastJoinAndLeave(t *testing.T) { return } - addr := &UDPAddr{ - IP: IPv4zero, - Port: 0, - } - // open a UDPConn - conn, err := ListenUDP("udp4", addr) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - // try to join group - mcast := IPv4(224, 0, 0, 254) - err = conn.JoinGroup(mcast) - if err != nil { - t.Fatal(err) - } - - // try to leave group - err = conn.LeaveGroup(mcast) - if err != nil { - t.Fatal(err) - } -} - -func TestJoinFailureWithIPv6Address(t *testing.T) { - if !*multicast { - t.Logf("test disabled; use --multicast to enable") - return - } - addr := &UDPAddr{ - IP: IPv4zero, - Port: 0, - } - - // open a UDPConn - conn, err := ListenUDP("udp4", addr) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - // try to join group - mcast := ParseIP("ff02::1") - err = conn.JoinGroup(mcast) - if err == nil { - t.Fatal("JoinGroup succeeded, should fail") + for _, tt := range joinAndLeaveGroupUDPTests { + var ( + ifi *Interface + found bool + ) + if tt.ipv6 && (!supportsIPv6 || os.Getuid() != 0) { + continue + } + ift, err := Interfaces() + if err != nil { + t.Fatalf("Interfaces() failed: %v", err) + } + for _, x := range ift { + if x.Flags&tt.flags == tt.flags { + ifi = &x + break + } + } + if ifi == nil { + t.Logf("an appropriate multicast interface not found") + return + } + c, err := ListenUDP(tt.net, &UDPAddr{IP: tt.laddr}) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if err := c.JoinGroup(ifi, tt.gaddr); err != nil { + t.Fatal(err) + } + ifmat, err := ifi.MulticastAddrs() + if err != nil { + t.Fatalf("MulticastAddrs() failed: %v", err) + } + for _, ifma := range ifmat { + if ifma.(*IPAddr).IP.Equal(tt.gaddr) { + found = true + break + } + } + if !found { + t.Fatalf("%q not found in RIB", tt.gaddr.String()) + } + if err := c.LeaveGroup(ifi, tt.gaddr); err != nil { + t.Fatal(err) + } } - t.Logf("%s", err) } diff --git a/libgo/go/net/net_test.go b/libgo/go/net/net_test.go index dc0d49d..b4a6e1f 100644 --- a/libgo/go/net/net_test.go +++ b/libgo/go/net/net_test.go @@ -6,6 +6,7 @@ package net import ( "flag" + "os" "regexp" "testing" ) @@ -61,6 +62,8 @@ var dialErrorTests = []DialErrorTest{ }, } +var duplicateErrorPattern = `dial (.*) dial (.*)` + func TestDialError(t *testing.T) { if !*runErrorTest { t.Logf("test disabled; use --run_error_test to enable") @@ -80,6 +83,10 @@ func TestDialError(t *testing.T) { if !match { t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern) } + match, _ = regexp.MatchString(duplicateErrorPattern, s) + if match { + t.Errorf("#%d: %q, duplicate error return from Dial", i, s) + } } } @@ -119,3 +126,46 @@ func TestReverseAddress(t *testing.T) { } } } + +func TestShutdown(t *testing.T) { + l, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = Listen("tcp6", "[::1]:0"); err != nil { + t.Fatalf("ListenTCP on :0: %v", err) + } + } + + go func() { + c, err := l.Accept() + if err != nil { + t.Fatalf("Accept: %v", err) + } + var buf [10]byte + n, err := c.Read(buf[:]) + if n != 0 || err != os.EOF { + t.Fatalf("server Read = %d, %v; want 0, os.EOF", n, err) + } + c.Write([]byte("response")) + c.Close() + }() + + c, err := Dial("tcp", l.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + + err = c.(*TCPConn).CloseWrite() + if err != nil { + t.Fatalf("CloseWrite: %v", err) + } + var buf [10]byte + n, err := c.Read(buf[:]) + if err != nil { + t.Fatalf("client Read: %d, %v", n, err) + } + got := string(buf[:n]) + if got != "response" { + t.Errorf("read = %q, want \"response\"", got) + } +} diff --git a/libgo/go/net/newpollserver.go b/libgo/go/net/newpollserver.go index 4272087..3c9a6da 100644 --- a/libgo/go/net/newpollserver.go +++ b/libgo/go/net/newpollserver.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package net import ( diff --git a/libgo/go/net/parse.go b/libgo/go/net/parse.go index de46830..0d30a7a 100644 --- a/libgo/go/net/parse.go +++ b/libgo/go/net/parse.go @@ -159,6 +159,18 @@ func xtoi(s string, i0 int) (n int, i int, ok bool) { return n, i, true } +// xtoi2 converts the next two hex digits of s into a byte. +// If s is longer than 2 bytes then the third byte must be e. +// If the first two bytes of s are not hex digits or the third byte +// does not match e, false is returned. +func xtoi2(s string, e byte) (byte, bool) { + if len(s) > 2 && s[2] != e { + return 0, false + } + n, ei, ok := xtoi(s[:2], 0) + return byte(n), ok && ei == 2 +} + // Integer to decimal. func itoa(i int) string { var buf [30]byte @@ -181,6 +193,37 @@ func itoa(i int) string { return string(buf[n:]) } +// Convert i to decimal string. +func itod(i uint) string { + if i == 0 { + return "0" + } + + // Assemble decimal in reverse order. + var b [32]byte + bp := len(b) + for ; i > 0; i /= 10 { + bp-- + b[bp] = byte(i%10) + '0' + } + + return string(b[bp:]) +} + +// Convert i to hexadecimal string. +func itox(i uint, min int) string { + // Assemble hexadecimal in reverse order. + var b [32]byte + bp := len(b) + for ; i > 0 || min > 0; i /= 16 { + bp-- + b[bp] = "0123456789abcdef"[byte(i%16)] + min-- + } + + return string(b[bp:]) +} + // Number of occurrences of b in s. func count(s string, b byte) int { n := 0 diff --git a/libgo/go/net/port.go b/libgo/go/net/port.go index 8f8327a..a8ca60c 100644 --- a/libgo/go/net/port.go +++ b/libgo/go/net/port.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + // Read system port mappings from /etc/services package net diff --git a/libgo/go/net/sendfile_stub.go b/libgo/go/net/sendfile_stub.go index 43e8104..c55be6c 100644 --- a/libgo/go/net/sendfile_stub.go +++ b/libgo/go/net/sendfile_stub.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd openbsd + package net import ( diff --git a/libgo/go/net/sendfile_windows.go b/libgo/go/net/sendfile_windows.go index 3772eee..d9c2f53 100644 --- a/libgo/go/net/sendfile_windows.go +++ b/libgo/go/net/sendfile_windows.go @@ -54,7 +54,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err os.Error, handled bool) defer c.decref() var o sendfileOp - o.Init(c) + o.Init(c, 'w') o.n = uint32(n) o.src = f.Fd() done, err := iosrv.ExecIO(&o, 0) diff --git a/libgo/go/net/server_test.go b/libgo/go/net/server_test.go index 7d7f7fc..a2ff218 100644 --- a/libgo/go/net/server_test.go +++ b/libgo/go/net/server_test.go @@ -115,7 +115,9 @@ func doTest(t *testing.T, network, listenaddr, dialaddr string) { } func TestTCPServer(t *testing.T) { - doTest(t, "tcp", "", "127.0.0.1") + if syscall.OS != "openbsd" { + doTest(t, "tcp", "", "127.0.0.1") + } doTest(t, "tcp", "0.0.0.0", "127.0.0.1") doTest(t, "tcp", "127.0.0.1", "127.0.0.1") doTest(t, "tcp4", "", "127.0.0.1") diff --git a/libgo/go/net/sock.go b/libgo/go/net/sock.go index 821716e..2359014 100644 --- a/libgo/go/net/sock.go +++ b/libgo/go/net/sock.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + // Sockets package net @@ -50,6 +52,7 @@ func socket(net string, f, p, t int, la, ra syscall.Sockaddr, toAddr func(syscal if ra != nil { if err = fd.connect(ra); err != nil { + closesocket(s) fd.Close() return nil, err } diff --git a/libgo/go/net/sock_bsd.go b/libgo/go/net/sock_bsd.go index 5fd5207..c59802f 100644 --- a/libgo/go/net/sock_bsd.go +++ b/libgo/go/net/sock_bsd.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd + // Sockets for BSD variants package net diff --git a/libgo/go/net/tcpsock_posix.go b/libgo/go/net/tcpsock_posix.go index 5560301..740a63d 100644 --- a/libgo/go/net/tcpsock_posix.go +++ b/libgo/go/net/tcpsock_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + // TCP sockets package net @@ -12,6 +14,11 @@ import ( "syscall" ) +// BUG(rsc): On OpenBSD, listening on the "tcp" network does not listen for +// both IPv4 and IPv6 connections. This is due to the fact that IPv4 traffic +// will not be routed to an IPv6 socket - two separate sockets are required +// if both AFs are to be supported. See inet6(4) on OpenBSD for details. + func sockaddrToTCP(sa syscall.Sockaddr) Addr { switch sa := sa.(type) { case *syscall.SockaddrInet4: @@ -23,7 +30,7 @@ func sockaddrToTCP(sa syscall.Sockaddr) Addr { } func (a *TCPAddr) family() int { - if a == nil || len(a.IP) <= 4 { + if a == nil || len(a.IP) <= IPv4len { return syscall.AF_INET } if a.IP.To4() != nil { @@ -93,6 +100,24 @@ func (c *TCPConn) Close() os.Error { return err } +// CloseRead shuts down the reading side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseRead() os.Error { + if !c.ok() { + return os.EINVAL + } + return c.fd.CloseRead() +} + +// CloseWrite shuts down the writing side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseWrite() os.Error { + if !c.ok() { + return os.EINVAL + } + return c.fd.CloseWrite() +} + // LocalAddr returns the local network address, a *TCPAddr. func (c *TCPConn) LocalAddr() Addr { if !c.ok() { diff --git a/libgo/go/net/textproto/header.go b/libgo/go/net/textproto/header.go index 288deb2..7fb32f80 100644 --- a/libgo/go/net/textproto/header.go +++ b/libgo/go/net/textproto/header.go @@ -39,5 +39,5 @@ func (h MIMEHeader) Get(key string) string { // Del deletes the values associated with key. func (h MIMEHeader) Del(key string) { - h[CanonicalMIMEHeaderKey(key)] = nil, false + delete(h, CanonicalMIMEHeaderKey(key)) } diff --git a/libgo/go/net/textproto/pipeline.go b/libgo/go/net/textproto/pipeline.go index 8c25884..ca50edd 100644 --- a/libgo/go/net/textproto/pipeline.go +++ b/libgo/go/net/textproto/pipeline.go @@ -108,7 +108,7 @@ func (s *sequencer) End(id uint) { } c, ok := s.wait[id] if ok { - s.wait[id] = nil, false + delete(s.wait, id) } s.mu.Unlock() if ok { diff --git a/libgo/go/net/textproto/reader.go b/libgo/go/net/textproto/reader.go index ce0ddc7..ece9a99 100644 --- a/libgo/go/net/textproto/reader.go +++ b/libgo/go/net/textproto/reader.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "strconv" + "strings" ) // BUG(rsc): To let callers manage exposure to denial of service @@ -182,6 +183,10 @@ func (r *Reader) readCodeLine(expectCode int) (code int, continued bool, message if err != nil { return } + return parseCodeLine(line, expectCode) +} + +func parseCodeLine(line string, expectCode int) (code int, continued bool, message string, err os.Error) { if len(line) < 4 || line[3] != ' ' && line[3] != '-' { err = ProtocolError("short response: " + line) return @@ -224,15 +229,20 @@ func (r *Reader) ReadCodeLine(expectCode int) (code int, message string, err os. return } -// ReadResponse reads a multi-line response of the form +// ReadResponse reads a multi-line response of the form: +// // code-message line 1 // code-message line 2 // ... // code message line n -// where code is a 3-digit status code. Each line should have the same code. -// The response is terminated by a line that uses a space between the code and -// the message line rather than a dash. Each line in message is separated by -// a newline (\n). +// +// where code is a 3-digit status code. The first line starts with the +// code and a hyphen. The response is terminated by a line that starts +// with the same code followed by a space. Each line in message is +// separated by a newline (\n). +// +// See page 36 of RFC 959 (http://www.ietf.org/rfc/rfc959.txt) for +// details. // // If the prefix of the status does not match the digits in expectCode, // ReadResponse returns with err set to &Error{code, message}. @@ -244,11 +254,18 @@ func (r *Reader) ReadCodeLine(expectCode int) (code int, message string, err os. func (r *Reader) ReadResponse(expectCode int) (code int, message string, err os.Error) { code, continued, message, err := r.readCodeLine(expectCode) for err == nil && continued { + line, err := r.ReadLine() + if err != nil { + return 0, "", err + } + var code2 int var moreMessage string - code2, continued, moreMessage, err = r.readCodeLine(expectCode) - if code != code2 { - err = ProtocolError("status code mismatch: " + strconv.Itoa(code) + ", " + strconv.Itoa(code2)) + code2, continued, moreMessage, err = parseCodeLine(line, expectCode) + if err != nil || code2 != code { + message += "\n" + strings.TrimRight(line, "\r\n") + continued = true + continue } message += "\n" + moreMessage } diff --git a/libgo/go/net/textproto/reader_test.go b/libgo/go/net/textproto/reader_test.go index 0658e58..23ebc3f 100644 --- a/libgo/go/net/textproto/reader_test.go +++ b/libgo/go/net/textproto/reader_test.go @@ -138,3 +138,56 @@ func TestReadMIMEHeader(t *testing.T) { t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want) } } + +type readResponseTest struct { + in string + inCode int + wantCode int + wantMsg string +} + +var readResponseTests = []readResponseTest{ + {"230-Anonymous access granted, restrictions apply\n" + + "Read the file README.txt,\n" + + "230 please", + 23, + 230, + "Anonymous access granted, restrictions apply\nRead the file README.txt,\n please", + }, + + {"230 Anonymous access granted, restrictions apply\n", + 23, + 230, + "Anonymous access granted, restrictions apply", + }, + + {"400-A\n400-B\n400 C", + 4, + 400, + "A\nB\nC", + }, + + {"400-A\r\n400-B\r\n400 C\r\n", + 4, + 400, + "A\nB\nC", + }, +} + +// See http://www.ietf.org/rfc/rfc959.txt page 36. +func TestRFC959Lines(t *testing.T) { + for i, tt := range readResponseTests { + r := reader(tt.in + "\nFOLLOWING DATA") + code, msg, err := r.ReadResponse(tt.inCode) + if err != nil { + t.Errorf("#%d: ReadResponse: %v", i, err) + continue + } + if code != tt.wantCode { + t.Errorf("#%d: code=%d, want %d", i, code, tt.wantCode) + } + if msg != tt.wantMsg { + t.Errorf("%#d: msg=%q, want %q", i, msg, tt.wantMsg) + } + } +} diff --git a/libgo/go/net/udpsock_plan9.go b/libgo/go/net/udpsock_plan9.go index bb71960..d5c6ccb 100644 --- a/libgo/go/net/udpsock_plan9.go +++ b/libgo/go/net/udpsock_plan9.go @@ -169,17 +169,18 @@ func ListenUDP(net string, laddr *UDPAddr) (c *UDPConn, err os.Error) { return &UDPConn{*l.plan9Conn()}, nil } -// JoinGroup joins the IPv4 multicast group named by addr. -// The UDPConn must use the "udp4" network. -func (c *UDPConn) JoinGroup(addr IP) os.Error { +// JoinGroup joins the IP multicast group named by addr on ifi, +// which specifies the interface to join. JoinGroup uses the +// default multicast interface if ifi is nil. +func (c *UDPConn) JoinGroup(ifi *Interface, addr IP) os.Error { if !c.ok() { return os.EINVAL } return os.EPLAN9 } -// LeaveGroup exits the IPv4 multicast group named by addr. -func (c *UDPConn) LeaveGroup(addr IP) os.Error { +// LeaveGroup exits the IP multicast group named by addr on ifi. +func (c *UDPConn) LeaveGroup(ifi *Interface, addr IP) os.Error { if !c.ok() { return os.EINVAL } diff --git a/libgo/go/net/udpsock_posix.go b/libgo/go/net/udpsock_posix.go index d4ea056..06298ee 100644 --- a/libgo/go/net/udpsock_posix.go +++ b/libgo/go/net/udpsock_posix.go @@ -2,11 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + // UDP sockets package net import ( + "bytes" "os" "syscall" ) @@ -22,7 +25,7 @@ func sockaddrToUDP(sa syscall.Sockaddr) Addr { } func (a *UDPAddr) family() int { - if a == nil || len(a.IP) <= 4 { + if a == nil || len(a.IP) <= IPv4len { return syscall.AF_INET } if a.IP.To4() != nil { @@ -252,43 +255,94 @@ func (c *UDPConn) BindToDevice(device string) os.Error { // Closing c does not affect f, and closing f does not affect c. func (c *UDPConn) File() (f *os.File, err os.Error) { return c.fd.dup() } -var errInvalidMulticast = os.NewError("invalid IPv4 multicast address") +// JoinGroup joins the IP multicast group named by addr on ifi, +// which specifies the interface to join. JoinGroup uses the +// default multicast interface if ifi is nil. +func (c *UDPConn) JoinGroup(ifi *Interface, addr IP) os.Error { + if !c.ok() { + return os.EINVAL + } + ip := addr.To4() + if ip != nil { + return joinIPv4GroupUDP(c, ifi, ip) + } + return joinIPv6GroupUDP(c, ifi, addr) +} -// JoinGroup joins the IPv4 multicast group named by addr. -// The UDPConn must use the "udp4" network. -func (c *UDPConn) JoinGroup(addr IP) os.Error { +// LeaveGroup exits the IP multicast group named by addr on ifi. +func (c *UDPConn) LeaveGroup(ifi *Interface, addr IP) os.Error { if !c.ok() { return os.EINVAL } ip := addr.To4() - if ip == nil { - return &OpError{"joingroup", "udp", &IPAddr{ip}, errInvalidMulticast} + if ip != nil { + return leaveIPv4GroupUDP(c, ifi, ip) } - mreq := &syscall.IPMreq{ - Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, + return leaveIPv6GroupUDP(c, ifi, addr) +} + +func joinIPv4GroupUDP(c *UDPConn, ifi *Interface, ip IP) os.Error { + mreq := &syscall.IPMreq{Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}} + if err := setIPv4InterfaceToJoin(mreq, ifi); err != nil { + return &OpError{"joinipv4group", "udp", &IPAddr{ip}, err} } - err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) - if err != nil { - return &OpError{"joingroup", "udp", &IPAddr{ip}, err} + if err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)); err != nil { + return &OpError{"joinipv4group", "udp", &IPAddr{ip}, err} } return nil } -// LeaveGroup exits the IPv4 multicast group named by addr. -func (c *UDPConn) LeaveGroup(addr IP) os.Error { - if !c.ok() { - return os.EINVAL +func leaveIPv4GroupUDP(c *UDPConn, ifi *Interface, ip IP) os.Error { + mreq := &syscall.IPMreq{Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}} + if err := setIPv4InterfaceToJoin(mreq, ifi); err != nil { + return &OpError{"leaveipv4group", "udp", &IPAddr{ip}, err} } - ip := addr.To4() - if ip == nil { - return &OpError{"leavegroup", "udp", &IPAddr{ip}, errInvalidMulticast} + if err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq)); err != nil { + return &OpError{"leaveipv4group", "udp", &IPAddr{ip}, err} } - mreq := &syscall.IPMreq{ - Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, + return nil +} + +func setIPv4InterfaceToJoin(mreq *syscall.IPMreq, ifi *Interface) os.Error { + if ifi == nil { + return nil } - err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq)) + ifat, err := ifi.Addrs() if err != nil { - return &OpError{"leavegroup", "udp", &IPAddr{ip}, err} + return err + } + for _, ifa := range ifat { + if x := ifa.(*IPAddr).IP.To4(); x != nil { + copy(mreq.Interface[:], x) + break + } + } + if bytes.Equal(mreq.Multiaddr[:], IPv4zero) { + return os.EINVAL + } + return nil +} + +func joinIPv6GroupUDP(c *UDPConn, ifi *Interface, ip IP) os.Error { + mreq := &syscall.IPv6Mreq{} + copy(mreq.Multiaddr[:], ip) + if ifi != nil { + mreq.Interface = uint32(ifi.Index) + } + if err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPv6Mreq(c.fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_JOIN_GROUP, mreq)); err != nil { + return &OpError{"joinipv6group", "udp", &IPAddr{ip}, err} + } + return nil +} + +func leaveIPv6GroupUDP(c *UDPConn, ifi *Interface, ip IP) os.Error { + mreq := &syscall.IPv6Mreq{} + copy(mreq.Multiaddr[:], ip) + if ifi != nil { + mreq.Interface = uint32(ifi.Index) + } + if err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPv6Mreq(c.fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_LEAVE_GROUP, mreq)); err != nil { + return &OpError{"leaveipv6group", "udp", &IPAddr{ip}, err} } return nil } diff --git a/libgo/go/net/unixsock_posix.go b/libgo/go/net/unixsock_posix.go index 38c6fe9..fccf018 100644 --- a/libgo/go/net/unixsock_posix.go +++ b/libgo/go/net/unixsock_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + // Unix domain sockets package net diff --git a/libgo/go/netchan/common.go b/libgo/go/old/netchan/common.go index ac1ca12..ac1ca12 100644 --- a/libgo/go/netchan/common.go +++ b/libgo/go/old/netchan/common.go diff --git a/libgo/go/netchan/export.go b/libgo/go/old/netchan/export.go index 7df7365..99d5d7e 100644 --- a/libgo/go/netchan/export.go +++ b/libgo/go/old/netchan/export.go @@ -314,7 +314,7 @@ func (exp *Exporter) addClient(conn io.ReadWriter) *expClient { // delClient forgets the client existed func (exp *Exporter) delClient(client *expClient) { exp.mu.Lock() - exp.clients[client] = false, false + delete(exp.clients, client) exp.mu.Unlock() } @@ -388,7 +388,7 @@ func (exp *Exporter) Hangup(name string) os.Error { exp.mu.Lock() chDir, ok := exp.names[name] if ok { - exp.names[name] = nil, false + delete(exp.names, name) } // TODO drop all instances of channel from client sets exp.mu.Unlock() diff --git a/libgo/go/netchan/import.go b/libgo/go/old/netchan/import.go index ec17d97..5a459e0 100644 --- a/libgo/go/netchan/import.go +++ b/libgo/go/old/netchan/import.go @@ -256,8 +256,8 @@ func (imp *Importer) Hangup(name string) os.Error { if nc == nil { return os.NewError("netchan import: hangup: no such channel: " + name) } - imp.names[name] = nil, false - imp.chans[nc.id] = nil, false + delete(imp.names, name) + delete(imp.chans, nc.id) nc.close() return nil } diff --git a/libgo/go/netchan/netchan_test.go b/libgo/go/old/netchan/netchan_test.go index 8c0f9a6..8c0f9a6 100644 --- a/libgo/go/netchan/netchan_test.go +++ b/libgo/go/old/netchan/netchan_test.go diff --git a/libgo/go/exp/regexp/all_test.go b/libgo/go/old/regexp/all_test.go index 77f32ca..71edc4d 100644 --- a/libgo/go/exp/regexp/all_test.go +++ b/libgo/go/old/regexp/all_test.go @@ -24,13 +24,13 @@ var good_re = []string{ `[a-z]`, `[a-abc-c\-\]\[]`, `[a-z]+`, + `[]`, `[abc]`, `[^1234]`, `[^\n]`, `\!\\`, } -/* type stringError struct { re string err os.Error @@ -51,7 +51,6 @@ var bad_re = []stringError{ {`a??`, ErrBadClosure}, {`\x`, ErrBadBackslash}, } -*/ func compileTest(t *testing.T, expr string, error os.Error) *Regexp { re, err := Compile(expr) @@ -67,13 +66,11 @@ func TestGoodCompile(t *testing.T) { } } -/* func TestBadCompile(t *testing.T) { for i := 0; i < len(bad_re); i++ { compileTest(t, bad_re[i].re, bad_re[i].err) } } -*/ func matchTest(t *testing.T, test *FindTest) { re := compileTest(t, test.pat, nil) @@ -243,7 +240,7 @@ var metaTests = []MetaTest{ {`foo`, `foo`, `foo`, true}, {`foo\.\$`, `foo\\\.\\\$`, `foo.$`, true}, // has meta but no operator {`foo.\$`, `foo\.\\\$`, `foo`, false}, // has escaped operators and real operators - {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[\{\]\}\\\|,<\.>/\?~`, `!@#`, false}, + {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[{\]}\\\|,<\.>/\?~`, `!@#`, false}, } func TestQuoteMeta(t *testing.T) { diff --git a/libgo/go/exp/regexp/find_test.go b/libgo/go/old/regexp/find_test.go index dddc348..83b249e 100644 --- a/libgo/go/exp/regexp/find_test.go +++ b/libgo/go/old/regexp/find_test.go @@ -58,8 +58,8 @@ var findTests = []FindTest{ {`(([^xyz]*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 3, 4)}, {`((a|b|c)*(d))`, "abcd", build(1, 0, 4, 0, 4, 2, 3, 3, 4)}, {`(((a|b|c)*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 2, 3, 3, 4)}, - {`\a\f\n\r\t\v`, "\a\f\n\r\t\v", build(1, 0, 6)}, - {`[\a\f\n\r\t\v]+`, "\a\f\n\r\t\v", build(1, 0, 6)}, + {`\a\b\f\n\r\t\v`, "\a\b\f\n\r\t\v", build(1, 0, 7)}, + {`[\a\b\f\n\r\t\v]+`, "\a\b\f\n\r\t\v", build(1, 0, 7)}, {`a*(|(b))c*`, "aacc", build(1, 0, 4, 2, 2, -1, -1)}, {`(.*).*`, "ab", build(1, 0, 2, 0, 2)}, @@ -209,7 +209,7 @@ func TestFindAll(t *testing.T) { case test.matches == nil && result != nil: t.Errorf("expected no match; got one: %s", test) case test.matches != nil && result == nil: - t.Fatalf("expected match; got none: %s", test) + t.Errorf("expected match; got none: %s", test) case test.matches != nil && result != nil: if len(test.matches) != len(result) { t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) diff --git a/libgo/go/exp/regexp/regexp.go b/libgo/go/old/regexp/regexp.go index 1b75900..e8d4c08 100644 --- a/libgo/go/exp/regexp/regexp.go +++ b/libgo/go/old/regexp/regexp.go @@ -3,12 +3,27 @@ // Package regexp implements a simple regular expression library. // -// The syntax of the regular expressions accepted is the same -// general syntax used by Perl, Python, and other languages. -// More precisely, it is the syntax accepted by RE2 and described at -// http://code.google.com/p/re2/wiki/Syntax, except for \C. +// The syntax of the regular expressions accepted is: // -// All characters are UTF-8-encoded code points. +// regexp: +// concatenation { '|' concatenation } +// concatenation: +// { closure } +// closure: +// term [ '*' | '+' | '?' ] +// term: +// '^' +// '$' +// '.' +// character +// '[' [ '^' ] { character-range } ']' +// '(' regexp ')' +// character-range: +// character [ '-' character ] +// +// All characters are UTF-8-encoded code points. Backslashes escape special +// characters, including inside character classes. The standard Go character +// escapes are also recognized: \a \b \f \n \r \t \v. // // There are 16 methods of Regexp that match a regular expression and identify // the matched text. Their names are matched by this regular expression: @@ -55,11 +70,9 @@ package regexp import ( "bytes" - "exp/regexp/syntax" "io" "os" "strings" - "sync" "utf8" ) @@ -72,22 +85,528 @@ func (e Error) String() string { return string(e) } +// Error codes returned by failures to parse an expression. +var ( + ErrInternal = Error("regexp: internal error") + ErrUnmatchedLpar = Error("regexp: unmatched '('") + ErrUnmatchedRpar = Error("regexp: unmatched ')'") + ErrUnmatchedLbkt = Error("regexp: unmatched '['") + ErrUnmatchedRbkt = Error("regexp: unmatched ']'") + ErrBadRange = Error("regexp: bad range in character class") + ErrExtraneousBackslash = Error("regexp: extraneous backslash") + ErrBadClosure = Error("regexp: repeated closure (**, ++, etc.)") + ErrBareClosure = Error("regexp: closure applies to nothing") + ErrBadBackslash = Error("regexp: illegal backslash escape") +) + +const ( + iStart = iota // beginning of program + iEnd // end of program: success + iBOT // '^' beginning of text + iEOT // '$' end of text + iChar // 'a' regular character + iCharClass // [a-z] character class + iAny // '.' any character including newline + iNotNL // [^\n] special case: any character but newline + iBra // '(' parenthesized expression: 2*braNum for left, 2*braNum+1 for right + iAlt // '|' alternation + iNop // do nothing; makes it easy to link without patching +) + +// An instruction executed by the NFA +type instr struct { + kind int // the type of this instruction: iChar, iAny, etc. + index int // used only in debugging; could be eliminated + next *instr // the instruction to execute after this one + // Special fields valid only for some items. + char int // iChar + braNum int // iBra, iEbra + cclass *charClass // iCharClass + left *instr // iAlt, other branch +} + +func (i *instr) print() { + switch i.kind { + case iStart: + print("start") + case iEnd: + print("end") + case iBOT: + print("bot") + case iEOT: + print("eot") + case iChar: + print("char ", string(i.char)) + case iCharClass: + i.cclass.print() + case iAny: + print("any") + case iNotNL: + print("notnl") + case iBra: + if i.braNum&1 == 0 { + print("bra", i.braNum/2) + } else { + print("ebra", i.braNum/2) + } + case iAlt: + print("alt(", i.left.index, ")") + case iNop: + print("nop") + } +} + // Regexp is the representation of a compiled regular expression. // The public interface is entirely through methods. // A Regexp is safe for concurrent use by multiple goroutines. type Regexp struct { - // read-only after Compile - expr string // as passed to Compile - prog *syntax.Prog // compiled program - prefix string // required prefix in unanchored matches - prefixBytes []byte // prefix, as a []byte - prefixComplete bool // prefix is the entire regexp - prefixRune int // first rune in prefix - cond syntax.EmptyOp // empty-width conditions required at start of match + expr string // the original expression + prefix string // initial plain text string + prefixBytes []byte // initial plain text bytes + inst []*instr + start *instr // first instruction of machine + prefixStart *instr // where to start if there is a prefix + nbra int // number of brackets in expression, for subexpressions +} + +type charClass struct { + negate bool // is character class negated? ([^a-z]) + // slice of int, stored pairwise: [a-z] is (a,z); x is (x,x): + ranges []int + cmin, cmax int +} + +func (cclass *charClass) print() { + print("charclass") + if cclass.negate { + print(" (negated)") + } + for i := 0; i < len(cclass.ranges); i += 2 { + l := cclass.ranges[i] + r := cclass.ranges[i+1] + if l == r { + print(" [", string(l), "]") + } else { + print(" [", string(l), "-", string(r), "]") + } + } +} + +func (cclass *charClass) addRange(a, b int) { + // range is a through b inclusive + cclass.ranges = append(cclass.ranges, a, b) + if a < cclass.cmin { + cclass.cmin = a + } + if b > cclass.cmax { + cclass.cmax = b + } +} + +func (cclass *charClass) matches(c int) bool { + if c < cclass.cmin || c > cclass.cmax { + return cclass.negate + } + ranges := cclass.ranges + for i := 0; i < len(ranges); i = i + 2 { + if ranges[i] <= c && c <= ranges[i+1] { + return !cclass.negate + } + } + return cclass.negate +} + +func newCharClass() *instr { + i := &instr{kind: iCharClass} + i.cclass = new(charClass) + i.cclass.ranges = make([]int, 0, 4) + i.cclass.cmin = 0x10FFFF + 1 // MaxRune + 1 + i.cclass.cmax = -1 + return i +} + +func (re *Regexp) add(i *instr) *instr { + i.index = len(re.inst) + re.inst = append(re.inst, i) + return i +} + +type parser struct { + re *Regexp + nlpar int // number of unclosed lpars + pos int + ch int +} + +func (p *parser) error(err Error) { + panic(err) +} + +const endOfText = -1 + +func (p *parser) c() int { return p.ch } + +func (p *parser) nextc() int { + if p.pos >= len(p.re.expr) { + p.ch = endOfText + } else { + c, w := utf8.DecodeRuneInString(p.re.expr[p.pos:]) + p.ch = c + p.pos += w + } + return p.ch +} + +func newParser(re *Regexp) *parser { + p := new(parser) + p.re = re + p.nextc() // load p.ch + return p +} + +func special(c int) bool { + for _, r := range `\.+*?()|[]^$` { + if c == r { + return true + } + } + return false +} - // cache of machines for running regexp - mu sync.Mutex - machine []*machine +func ispunct(c int) bool { + for _, r := range "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" { + if c == r { + return true + } + } + return false +} + +var escapes = []byte("abfnrtv") +var escaped = []byte("\a\b\f\n\r\t\v") + +func escape(c int) int { + for i, b := range escapes { + if int(b) == c { + return i + } + } + return -1 +} + +func (p *parser) checkBackslash() int { + c := p.c() + if c == '\\' { + c = p.nextc() + switch { + case c == endOfText: + p.error(ErrExtraneousBackslash) + case ispunct(c): + // c is as delivered + case escape(c) >= 0: + c = int(escaped[escape(c)]) + default: + p.error(ErrBadBackslash) + } + } + return c +} + +func (p *parser) charClass() *instr { + i := newCharClass() + cc := i.cclass + if p.c() == '^' { + cc.negate = true + p.nextc() + } + left := -1 + for { + switch c := p.c(); c { + case ']', endOfText: + if left >= 0 { + p.error(ErrBadRange) + } + // Is it [^\n]? + if cc.negate && len(cc.ranges) == 2 && + cc.ranges[0] == '\n' && cc.ranges[1] == '\n' { + nl := &instr{kind: iNotNL} + p.re.add(nl) + return nl + } + // Special common case: "[a]" -> "a" + if !cc.negate && len(cc.ranges) == 2 && cc.ranges[0] == cc.ranges[1] { + c := &instr{kind: iChar, char: cc.ranges[0]} + p.re.add(c) + return c + } + p.re.add(i) + return i + case '-': // do this before backslash processing + p.error(ErrBadRange) + default: + c = p.checkBackslash() + p.nextc() + switch { + case left < 0: // first of pair + if p.c() == '-' { // range + p.nextc() + left = c + } else { // single char + cc.addRange(c, c) + } + case left <= c: // second of pair + cc.addRange(left, c) + left = -1 + default: + p.error(ErrBadRange) + } + } + } + panic("unreachable") +} + +func (p *parser) term() (start, end *instr) { + switch c := p.c(); c { + case '|', endOfText: + return nil, nil + case '*', '+', '?': + p.error(ErrBareClosure) + case ')': + if p.nlpar == 0 { + p.error(ErrUnmatchedRpar) + } + return nil, nil + case ']': + p.error(ErrUnmatchedRbkt) + case '^': + p.nextc() + start = p.re.add(&instr{kind: iBOT}) + return start, start + case '$': + p.nextc() + start = p.re.add(&instr{kind: iEOT}) + return start, start + case '.': + p.nextc() + start = p.re.add(&instr{kind: iAny}) + return start, start + case '[': + p.nextc() + start = p.charClass() + if p.c() != ']' { + p.error(ErrUnmatchedLbkt) + } + p.nextc() + return start, start + case '(': + p.nextc() + p.nlpar++ + p.re.nbra++ // increment first so first subexpr is \1 + nbra := p.re.nbra + start, end = p.regexp() + if p.c() != ')' { + p.error(ErrUnmatchedLpar) + } + p.nlpar-- + p.nextc() + bra := &instr{kind: iBra, braNum: 2 * nbra} + p.re.add(bra) + ebra := &instr{kind: iBra, braNum: 2*nbra + 1} + p.re.add(ebra) + if start == nil { + if end == nil { + p.error(ErrInternal) + return + } + start = ebra + } else { + end.next = ebra + } + bra.next = start + return bra, ebra + default: + c = p.checkBackslash() + p.nextc() + start = &instr{kind: iChar, char: c} + p.re.add(start) + return start, start + } + panic("unreachable") +} + +func (p *parser) closure() (start, end *instr) { + start, end = p.term() + if start == nil { + return + } + switch p.c() { + case '*': + // (start,end)*: + alt := &instr{kind: iAlt} + p.re.add(alt) + end.next = alt // after end, do alt + alt.left = start // alternate brach: return to start + start = alt // alt becomes new (start, end) + end = alt + case '+': + // (start,end)+: + alt := &instr{kind: iAlt} + p.re.add(alt) + end.next = alt // after end, do alt + alt.left = start // alternate brach: return to start + end = alt // start is unchanged; end is alt + case '?': + // (start,end)?: + alt := &instr{kind: iAlt} + p.re.add(alt) + nop := &instr{kind: iNop} + p.re.add(nop) + alt.left = start // alternate branch is start + alt.next = nop // follow on to nop + end.next = nop // after end, go to nop + start = alt // start is now alt + end = nop // end is nop pointed to by both branches + default: + return + } + switch p.nextc() { + case '*', '+', '?': + p.error(ErrBadClosure) + } + return +} + +func (p *parser) concatenation() (start, end *instr) { + for { + nstart, nend := p.closure() + switch { + case nstart == nil: // end of this concatenation + if start == nil { // this is the empty string + nop := p.re.add(&instr{kind: iNop}) + return nop, nop + } + return + case start == nil: // this is first element of concatenation + start, end = nstart, nend + default: + end.next = nstart + end = nend + } + } + panic("unreachable") +} + +func (p *parser) regexp() (start, end *instr) { + start, end = p.concatenation() + for { + switch p.c() { + default: + return + case '|': + p.nextc() + nstart, nend := p.concatenation() + alt := &instr{kind: iAlt} + p.re.add(alt) + alt.left = start + alt.next = nstart + nop := &instr{kind: iNop} + p.re.add(nop) + end.next = nop + nend.next = nop + start, end = alt, nop + } + } + panic("unreachable") +} + +func unNop(i *instr) *instr { + for i.kind == iNop { + i = i.next + } + return i +} + +func (re *Regexp) eliminateNops() { + for _, inst := range re.inst { + if inst.kind == iEnd { + continue + } + inst.next = unNop(inst.next) + if inst.kind == iAlt { + inst.left = unNop(inst.left) + } + } +} + +func (re *Regexp) dump() { + print("prefix <", re.prefix, ">\n") + for _, inst := range re.inst { + print(inst.index, ": ") + inst.print() + if inst.kind != iEnd { + print(" -> ", inst.next.index) + } + print("\n") + } +} + +func (re *Regexp) doParse() { + p := newParser(re) + start := &instr{kind: iStart} + re.add(start) + s, e := p.regexp() + start.next = s + re.start = start + e.next = re.add(&instr{kind: iEnd}) + + if debug { + re.dump() + println() + } + + re.eliminateNops() + if debug { + re.dump() + println() + } + re.setPrefix() + if debug { + re.dump() + println() + } +} + +// Extract regular text from the beginning of the pattern, +// possibly after a leading iBOT. +// That text can be used by doExecute to speed up matching. +func (re *Regexp) setPrefix() { + var b []byte + var utf = make([]byte, utf8.UTFMax) + var inst *instr + // First instruction is start; skip that. Also skip any initial iBOT. + inst = re.inst[0].next + for inst.kind == iBOT { + inst = inst.next + } +Loop: + for ; inst.kind != iEnd; inst = inst.next { + // stop if this is not a char + if inst.kind != iChar { + break + } + // stop if this char can be followed by a match for an empty string, + // which includes closures, ^, and $. + switch inst.next.kind { + case iBOT, iEOT, iAlt: + break Loop + } + n := utf8.EncodeRune(utf, inst.char) + b = append(b, utf[0:n]...) + } + // point prefixStart instruction to first non-CHAR after prefix + re.prefixStart = inst + re.prefixBytes = b + re.prefix = string(b) } // String returns the source text used to compile the regular expression. @@ -97,55 +616,19 @@ func (re *Regexp) String() string { // Compile parses a regular expression and returns, if successful, a Regexp // object that can be used to match against text. -func Compile(expr string) (*Regexp, os.Error) { - re, err := syntax.Parse(expr, syntax.Perl) - if err != nil { - return nil, err - } - prog, err := syntax.Compile(re) - if err != nil { - return nil, err - } - regexp := &Regexp{ - expr: expr, - prog: prog, - } - regexp.prefix, regexp.prefixComplete = prog.Prefix() - if regexp.prefix != "" { - // TODO(rsc): Remove this allocation by adding - // IndexString to package bytes. - regexp.prefixBytes = []byte(regexp.prefix) - regexp.prefixRune, _ = utf8.DecodeRuneInString(regexp.prefix) - } - regexp.cond = prog.StartCond() - return regexp, nil -} - -// get returns a machine to use for matching re. -// It uses the re's machine cache if possible, to avoid -// unnecessary allocation. -func (re *Regexp) get() *machine { - re.mu.Lock() - if n := len(re.machine); n > 0 { - z := re.machine[n-1] - re.machine = re.machine[:n-1] - re.mu.Unlock() - return z - } - re.mu.Unlock() - z := progMachine(re.prog) - z.re = re - return z -} - -// put returns a machine to the re's machine cache. -// There is no attempt to limit the size of the cache, so it will -// grow to the maximum number of simultaneous matches -// run using re. (The cache empties when re gets garbage collected.) -func (re *Regexp) put(z *machine) { - re.mu.Lock() - re.machine = append(re.machine, z) - re.mu.Unlock() +func Compile(str string) (regexp *Regexp, error os.Error) { + regexp = new(Regexp) + // doParse will panic if there is a parse error. + defer func() { + if e := recover(); e != nil { + regexp = nil + error = e.(Error) // Will re-panic if error was not an Error, e.g. nil-pointer exception + } + }() + regexp.expr = str + regexp.inst = make([]*instr, 0, 10) + regexp.doParse() + return } // MustCompile is like Compile but panics if the expression cannot be parsed. @@ -160,13 +643,110 @@ func MustCompile(str string) *Regexp { } // NumSubexp returns the number of parenthesized subexpressions in this Regexp. -func (re *Regexp) NumSubexp() int { - // NumCap/2 because captures count ( and ) separately. - // -1 because NumCap counts $0 but NumSubexp does not. - return re.prog.NumCap/2 - 1 +func (re *Regexp) NumSubexp() int { return re.nbra } + +// The match arena allows us to reduce the garbage generated by tossing +// match vectors away as we execute. Matches are ref counted and returned +// to a free list when no longer active. Increases a simple benchmark by 22X. +type matchArena struct { + head *matchVec + len int // length of match vector + pos int + atBOT bool // whether we're at beginning of text + atEOT bool // whether we're at end of text } -const endOfText = -1 +type matchVec struct { + m []int // pairs of bracketing submatches. 0th is start,end + ref int + next *matchVec +} + +func (a *matchArena) new() *matchVec { + if a.head == nil { + const N = 10 + block := make([]matchVec, N) + for i := 0; i < N; i++ { + b := &block[i] + b.next = a.head + a.head = b + } + } + m := a.head + a.head = m.next + m.ref = 0 + if m.m == nil { + m.m = make([]int, a.len) + } + return m +} + +func (a *matchArena) free(m *matchVec) { + m.ref-- + if m.ref == 0 { + m.next = a.head + a.head = m + } +} + +func (a *matchArena) copy(m *matchVec) *matchVec { + m1 := a.new() + copy(m1.m, m.m) + return m1 +} + +func (a *matchArena) noMatch() *matchVec { + m := a.new() + for i := range m.m { + m.m[i] = -1 // no match seen; catches cases like "a(b)?c" on "ac" + } + m.ref = 1 + return m +} + +type state struct { + inst *instr // next instruction to execute + prefixed bool // this match began with a fixed prefix + match *matchVec +} + +// Append new state to to-do list. Leftmost-longest wins so avoid +// adding a state that's already active. The matchVec will be inc-ref'ed +// if it is assigned to a state. +func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matchVec) []state { + switch inst.kind { + case iBOT: + if a.atBOT { + s = a.addState(s, inst.next, prefixed, match) + } + return s + case iEOT: + if a.atEOT { + s = a.addState(s, inst.next, prefixed, match) + } + return s + case iBra: + match.m[inst.braNum] = a.pos + s = a.addState(s, inst.next, prefixed, match) + return s + } + l := len(s) + // States are inserted in order so it's sufficient to see if we have the same + // instruction; no need to see if existing match is earlier (it is). + for i := 0; i < l; i++ { + if s[i].inst == inst { + return s + } + } + s = append(s, state{inst, prefixed, match}) + match.ref++ + if inst.kind == iAlt { + s = a.addState(s, inst.left, prefixed, a.copy(match)) + // give other branch a copy of this match vector + s = a.addState(s, inst.next, prefixed, a.copy(match)) + } + return s +} // input abstracts different representations of the input text. It provides // one-character lookahead. @@ -270,31 +850,150 @@ func (i *inputReader) index(re *Regexp, pos int) int { return -1 } +// Search match starting from pos bytes into the input. +func (re *Regexp) doExecute(i input, pos int) []int { + var s [2][]state + s[0] = make([]state, 0, 10) + s[1] = make([]state, 0, 10) + in, out := 0, 1 + var final state + found := false + anchored := re.inst[0].next.kind == iBOT + if anchored && pos > 0 { + return nil + } + // fast check for initial plain substring + if i.canCheckPrefix() && re.prefix != "" { + advance := 0 + if anchored { + if !i.hasPrefix(re) { + return nil + } + } else { + advance = i.index(re, pos) + if advance == -1 { + return nil + } + } + pos += advance + } + // We look one character ahead so we can match $, which checks whether + // we are at EOT. + nextChar, nextWidth := i.step(pos) + arena := &matchArena{ + len: 2 * (re.nbra + 1), + pos: pos, + atBOT: pos == 0, + atEOT: nextChar == endOfText, + } + for c, startPos := 0, pos; c != endOfText; { + if !found && (pos == startPos || !anchored) { + // prime the pump if we haven't seen a match yet + match := arena.noMatch() + match.m[0] = pos + s[out] = arena.addState(s[out], re.start.next, false, match) + arena.free(match) // if addState saved it, ref was incremented + } else if len(s[out]) == 0 { + // machine has completed + break + } + in, out = out, in // old out state is new in state + // clear out old state + old := s[out] + for _, state := range old { + arena.free(state.match) + } + s[out] = old[0:0] // truncate state vector + c = nextChar + thisPos := pos + pos += nextWidth + nextChar, nextWidth = i.step(pos) + arena.atEOT = nextChar == endOfText + arena.atBOT = false + arena.pos = pos + for _, st := range s[in] { + switch st.inst.kind { + case iBOT: + case iEOT: + case iChar: + if c == st.inst.char { + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) + } + case iCharClass: + if st.inst.cclass.matches(c) { + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) + } + case iAny: + if c != endOfText { + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) + } + case iNotNL: + if c != endOfText && c != '\n' { + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) + } + case iBra: + case iAlt: + case iEnd: + // choose leftmost longest + if !found || // first + st.match.m[0] < final.match.m[0] || // leftmost + (st.match.m[0] == final.match.m[0] && thisPos > final.match.m[1]) { // longest + if final.match != nil { + arena.free(final.match) + } + final = st + final.match.ref++ + final.match.m[1] = thisPos + } + found = true + default: + st.inst.print() + panic("unknown instruction in execute") + } + } + } + if final.match == nil { + return nil + } + // if match found, back up start of match by width of prefix. + if final.prefixed && len(final.match.m) > 0 { + final.match.m[0] -= len(re.prefix) + } + return final.match.m +} + // LiteralPrefix returns a literal string that must begin any match // of the regular expression re. It returns the boolean true if the // literal string comprises the entire regular expression. func (re *Regexp) LiteralPrefix() (prefix string, complete bool) { - return re.prefix, re.prefixComplete + c := make([]int, len(re.inst)-2) // minus start and end. + // First instruction is start; skip that. + i := 0 + for inst := re.inst[0].next; inst.kind != iEnd; inst = inst.next { + // stop if this is not a char + if inst.kind != iChar { + return string(c[:i]), false + } + c[i] = inst.char + i++ + } + return string(c[:i]), true } // MatchReader returns whether the Regexp matches the text read by the // RuneReader. The return value is a boolean: true for match, false for no // match. func (re *Regexp) MatchReader(r io.RuneReader) bool { - return re.doExecute(newInputReader(r), 0, 0) != nil + return len(re.doExecute(newInputReader(r), 0)) > 0 } // MatchString returns whether the Regexp matches the string s. // The return value is a boolean: true for match, false for no match. -func (re *Regexp) MatchString(s string) bool { - return re.doExecute(newInputString(s), 0, 0) != nil -} +func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(newInputString(s), 0)) > 0 } // Match returns whether the Regexp matches the byte slice b. // The return value is a boolean: true for match, false for no match. -func (re *Regexp) Match(b []byte) bool { - return re.doExecute(newInputBytes(b), 0, 0) != nil -} +func (re *Regexp) Match(b []byte) bool { return len(re.doExecute(newInputBytes(b), 0)) > 0 } // MatchReader checks whether a textual regular expression matches the text // read by the RuneReader. More complicated queries need to use Compile and @@ -345,7 +1044,7 @@ func (re *Regexp) ReplaceAllStringFunc(src string, repl func(string) string) str searchPos := 0 // position where we next look for a match buf := new(bytes.Buffer) for searchPos <= len(src) { - a := re.doExecute(newInputString(src), searchPos, 2) + a := re.doExecute(newInputString(src), searchPos) if len(a) == 0 { break // no more matches } @@ -397,7 +1096,7 @@ func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { searchPos := 0 // position where we next look for a match buf := new(bytes.Buffer) for searchPos <= len(src) { - a := re.doExecute(newInputBytes(src), searchPos, 2) + a := re.doExecute(newInputBytes(src), searchPos) if len(a) == 0 { break // no more matches } @@ -433,12 +1132,6 @@ func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { return buf.Bytes() } -var specialBytes = []byte(`\.+*?()|[]{}^$`) - -func special(b byte) bool { - return bytes.IndexByte(specialBytes, b) >= 0 -} - // QuoteMeta returns a string that quotes all regular expression metacharacters // inside the argument text; the returned string is a regular expression matching // the literal text. For example, QuoteMeta(`[foo]`) returns `\[foo\]`. @@ -448,7 +1141,7 @@ func QuoteMeta(s string) string { // A byte loop is correct because all metacharacters are ASCII. j := 0 for i := 0; i < len(s); i++ { - if special(s[i]) { + if special(int(s[i])) { b[j] = '\\' j++ } @@ -474,7 +1167,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { } else { in = newInputBytes(b) } - matches := re.doExecute(in, pos, re.prog.NumCap) + matches := re.doExecute(in, pos) if len(matches) == 0 { break } @@ -514,7 +1207,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { // Find returns a slice holding the text of the leftmost match in b of the regular expression. // A return value of nil indicates no match. func (re *Regexp) Find(b []byte) []byte { - a := re.doExecute(newInputBytes(b), 0, 2) + a := re.doExecute(newInputBytes(b), 0) if a == nil { return nil } @@ -526,7 +1219,7 @@ func (re *Regexp) Find(b []byte) []byte { // b[loc[0]:loc[1]]. // A return value of nil indicates no match. func (re *Regexp) FindIndex(b []byte) (loc []int) { - a := re.doExecute(newInputBytes(b), 0, 2) + a := re.doExecute(newInputBytes(b), 0) if a == nil { return nil } @@ -539,7 +1232,7 @@ func (re *Regexp) FindIndex(b []byte) (loc []int) { // an empty string. Use FindStringIndex or FindStringSubmatch if it is // necessary to distinguish these cases. func (re *Regexp) FindString(s string) string { - a := re.doExecute(newInputString(s), 0, 2) + a := re.doExecute(newInputString(s), 0) if a == nil { return "" } @@ -551,7 +1244,7 @@ func (re *Regexp) FindString(s string) string { // itself is at s[loc[0]:loc[1]]. // A return value of nil indicates no match. func (re *Regexp) FindStringIndex(s string) []int { - a := re.doExecute(newInputString(s), 0, 2) + a := re.doExecute(newInputString(s), 0) if a == nil { return nil } @@ -563,7 +1256,7 @@ func (re *Regexp) FindStringIndex(s string) []int { // the RuneReader. The match itself is at s[loc[0]:loc[1]]. A return // value of nil indicates no match. func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { - a := re.doExecute(newInputReader(r), 0, 2) + a := re.doExecute(newInputReader(r), 0) if a == nil { return nil } @@ -576,7 +1269,7 @@ func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { // comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatch(b []byte) [][]byte { - a := re.doExecute(newInputBytes(b), 0, re.prog.NumCap) + a := re.doExecute(newInputBytes(b), 0) if a == nil { return nil } @@ -595,7 +1288,7 @@ func (re *Regexp) FindSubmatch(b []byte) [][]byte { // in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatchIndex(b []byte) []int { - return re.doExecute(newInputBytes(b), 0, re.prog.NumCap) + return re.doExecute(newInputBytes(b), 0) } // FindStringSubmatch returns a slice of strings holding the text of the @@ -604,7 +1297,7 @@ func (re *Regexp) FindSubmatchIndex(b []byte) []int { // package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatch(s string) []string { - a := re.doExecute(newInputString(s), 0, re.prog.NumCap) + a := re.doExecute(newInputString(s), 0) if a == nil { return nil } @@ -623,7 +1316,7 @@ func (re *Regexp) FindStringSubmatch(s string) []string { // 'Index' descriptions in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatchIndex(s string) []int { - return re.doExecute(newInputString(s), 0, re.prog.NumCap) + return re.doExecute(newInputString(s), 0) } // FindReaderSubmatchIndex returns a slice holding the index pairs @@ -632,7 +1325,7 @@ func (re *Regexp) FindStringSubmatchIndex(s string) []int { // by the 'Submatch' and 'Index' descriptions in the package comment. A // return value of nil indicates no match. func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { - return re.doExecute(newInputReader(r), 0, re.prog.NumCap) + return re.doExecute(newInputReader(r), 0) } const startSize = 10 // The size at which to start a slice in the 'All' routines. diff --git a/libgo/go/old/template/template_test.go b/libgo/go/old/template/template_test.go index eae8011..9595eb1 100644 --- a/libgo/go/old/template/template_test.go +++ b/libgo/go/old/template/template_test.go @@ -6,7 +6,6 @@ package template import ( "bytes" - "container/vector" "fmt" "io" "io/ioutil" @@ -42,7 +41,7 @@ type S struct { Empty []*T Emptystring string Null []*T - Vec *vector.Vector + Vec []interface{} True bool False bool Mp map[string]string @@ -497,9 +496,7 @@ func testAll(t *testing.T, parseFunc func(*Test) (*Template, os.Error)) { s.Pdata = []*T{&t1, &t2} s.Empty = []*T{} s.Null = nil - s.Vec = new(vector.Vector) - s.Vec.Push("elt1") - s.Vec.Push("elt2") + s.Vec = []interface{}{"elt1", "elt2"} s.True = true s.False = false s.Mp = make(map[string]string) diff --git a/libgo/go/os/dir_plan9.go b/libgo/go/os/dir_plan9.go index bbc2cb6..bf17005 100644 --- a/libgo/go/os/dir_plan9.go +++ b/libgo/go/os/dir_plan9.go @@ -295,6 +295,6 @@ func pstring(b []byte, s string) []byte { panic(NewError("string too long")) } b = pbit16(b, uint16(len(s))) - b = append(b, []byte(s)...) + b = append(b, s...) return b } diff --git a/libgo/go/os/dir_unix.go b/libgo/go/os/dir_unix.go index 7835ed5..5295933 100644 --- a/libgo/go/os/dir_unix.go +++ b/libgo/go/os/dir_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package os import ( diff --git a/libgo/go/os/env.go b/libgo/go/os/env.go index 3772c09..4844fa3 100644 --- a/libgo/go/os/env.go +++ b/libgo/go/os/env.go @@ -16,9 +16,9 @@ func Expand(s string, mapping func(string) string) string { i := 0 for j := 0; j < len(s); j++ { if s[j] == '$' && j+1 < len(s) { - buf = append(buf, []byte(s[i:j])...) + buf = append(buf, s[i:j]...) name, w := getShellName(s[j+1:]) - buf = append(buf, []byte(mapping(name))...) + buf = append(buf, mapping(name)...) j += w i = j + 1 } diff --git a/libgo/go/os/env_unix.go b/libgo/go/os/env_unix.go index 9cc0b03..8dd84ae 100644 --- a/libgo/go/os/env_unix.go +++ b/libgo/go/os/env_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + // Unix environment variables. package os diff --git a/libgo/go/os/error_plan9.go b/libgo/go/os/error_plan9.go index cacfc15..91ace6d 100644 --- a/libgo/go/os/error_plan9.go +++ b/libgo/go/os/error_plan9.go @@ -48,6 +48,7 @@ var ( EPERM = Eperm EISDIR = syscall.EISDIR + EBADF = NewError("bad file descriptor") ENAMETOOLONG = NewError("file name too long") ERANGE = NewError("math result not representable") EPIPE = NewError("Broken Pipe") diff --git a/libgo/go/os/error_posix.go b/libgo/go/os/error_posix.go index d43f178..9dc258a 100644 --- a/libgo/go/os/error_posix.go +++ b/libgo/go/os/error_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + package os import syscall "syscall" diff --git a/libgo/go/os/exec_posix.go b/libgo/go/os/exec_posix.go index f37bfab..035b156 100644 --- a/libgo/go/os/exec_posix.go +++ b/libgo/go/os/exec_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + package os import ( diff --git a/libgo/go/os/exec_unix.go b/libgo/go/os/exec_unix.go index 8a4b2e1..e1adb20 100644 --- a/libgo/go/os/exec_unix.go +++ b/libgo/go/os/exec_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package os import ( @@ -38,7 +40,8 @@ func (p *Process) Wait(options int) (w *Waitmsg, err Error) { if e != 0 { return nil, NewSyscallError("wait", e) } - if options&WSTOPPED == 0 { + // With WNOHANG pid is 0 if child has not exited. + if pid1 != 0 && options&WSTOPPED == 0 { p.done = true } w = new(Waitmsg) diff --git a/libgo/go/os/file_posix.go b/libgo/go/os/file_posix.go index 0791a0d..5269149 100644 --- a/libgo/go/os/file_posix.go +++ b/libgo/go/os/file_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd windows + package os import ( @@ -21,39 +23,6 @@ func epipecheck(file *File, e int) { } } -// Stat returns a FileInfo structure describing the named file and an error, if any. -// If name names a valid symbolic link, the returned FileInfo describes -// the file pointed at by the link and has fi.FollowedSymlink set to true. -// If name names an invalid symbolic link, the returned FileInfo describes -// the link itself and has fi.FollowedSymlink set to false. -func Stat(name string) (fi *FileInfo, err Error) { - var lstat, stat syscall.Stat_t - e := syscall.Lstat(name, &lstat) - if iserror(e) { - return nil, &PathError{"stat", name, Errno(e)} - } - statp := &lstat - if lstat.Mode&syscall.S_IFMT == syscall.S_IFLNK { - e := syscall.Stat(name, &stat) - if !iserror(e) { - statp = &stat - } - } - return fileInfoFromStat(name, new(FileInfo), &lstat, statp), nil -} - -// Lstat returns the FileInfo structure describing the named file and an -// error, if any. If the file is a symbolic link, the returned FileInfo -// describes the symbolic link. Lstat makes no attempt to follow the link. -func Lstat(name string) (fi *FileInfo, err Error) { - var stat syscall.Stat_t - e := syscall.Lstat(name, &stat) - if iserror(e) { - return nil, &PathError{"lstat", name, Errno(e)} - } - return fileInfoFromStat(name, new(FileInfo), &stat, &stat), nil -} - // Remove removes the named file or directory. func Remove(name string) Error { // System call interface forces us to know @@ -79,7 +48,7 @@ func Remove(name string) Error { // both errors will be ENOTDIR, so it's okay to // use the error from unlink. // For windows syscall.ENOTDIR is set - // to syscall.ERROR_DIRECTORY, hopefully it should + // to syscall.ERROR_PATH_NOT_FOUND, hopefully it should // do the trick. if e1 != syscall.ENOTDIR { e = e1 diff --git a/libgo/go/os/file_unix.go b/libgo/go/os/file_unix.go index 8243701..0ebaf23 100644 --- a/libgo/go/os/file_unix.go +++ b/libgo/go/os/file_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package os import ( @@ -100,6 +102,39 @@ func (file *File) Stat() (fi *FileInfo, err Error) { return fileInfoFromStat(file.name, new(FileInfo), &stat, &stat), nil } +// Stat returns a FileInfo structure describing the named file and an error, if any. +// If name names a valid symbolic link, the returned FileInfo describes +// the file pointed at by the link and has fi.FollowedSymlink set to true. +// If name names an invalid symbolic link, the returned FileInfo describes +// the link itself and has fi.FollowedSymlink set to false. +func Stat(name string) (fi *FileInfo, err Error) { + var lstat, stat syscall.Stat_t + e := syscall.Lstat(name, &lstat) + if iserror(e) { + return nil, &PathError{"stat", name, Errno(e)} + } + statp := &lstat + if lstat.Mode&syscall.S_IFMT == syscall.S_IFLNK { + e := syscall.Stat(name, &stat) + if !iserror(e) { + statp = &stat + } + } + return fileInfoFromStat(name, new(FileInfo), &lstat, statp), nil +} + +// Lstat returns the FileInfo structure describing the named file and an +// error, if any. If the file is a symbolic link, the returned FileInfo +// describes the symbolic link. Lstat makes no attempt to follow the link. +func Lstat(name string) (fi *FileInfo, err Error) { + var stat syscall.Stat_t + e := syscall.Lstat(name, &stat) + if iserror(e) { + return nil, &PathError{"lstat", name, Errno(e)} + } + return fileInfoFromStat(name, new(FileInfo), &stat, &stat), nil +} + // Readdir reads the contents of the directory associated with file and // returns an array of up to n FileInfo structures, as would be returned // by Lstat, in directory order. Subsequent calls on the same file will yield diff --git a/libgo/go/os/inotify/inotify_linux.go b/libgo/go/os/inotify/inotify_linux.go index 99fa516..ee3c75f 100644 --- a/libgo/go/os/inotify/inotify_linux.go +++ b/libgo/go/os/inotify/inotify_linux.go @@ -131,7 +131,7 @@ func (w *Watcher) RemoveWatch(path string) os.Error { if success == -1 { return os.NewSyscallError("inotify_rm_watch", errno) } - w.watches[path] = nil, false + delete(w.watches, path) return nil } diff --git a/libgo/go/os/os_test.go b/libgo/go/os/os_test.go index 3f7d306..3277fc2 100644 --- a/libgo/go/os/os_test.go +++ b/libgo/go/os/os_test.go @@ -918,6 +918,12 @@ func TestHostname(t *testing.T) { if syscall.OS == "windows" || syscall.OS == "plan9" { return } + + // TODO(jsing): Fix nametomib() on OpenBSD + if syscall.OS == "openbsd" { + return + } + // Check internal Hostname() against the output of /bin/hostname. // Allow that the internal Hostname returns a Fully Qualified Domain Name // and the /bin/hostname only returns the first component diff --git a/libgo/go/os/path.go b/libgo/go/os/path.go index a8dfce3..b190c51 100644 --- a/libgo/go/os/path.go +++ b/libgo/go/os/path.go @@ -68,7 +68,7 @@ func RemoveAll(path string) Error { // Otherwise, is this a directory we need to recurse into? dir, serr := Lstat(path) if serr != nil { - if serr, ok := serr.(*PathError); ok && serr.Error == ENOENT { + if serr, ok := serr.(*PathError); ok && (serr.Error == ENOENT || serr.Error == ENOTDIR) { return nil } return serr diff --git a/libgo/go/os/path_unix.go b/libgo/go/os/path_unix.go index 0d327cd..33045b6 100644 --- a/libgo/go/os/path_unix.go +++ b/libgo/go/os/path_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package os const ( diff --git a/libgo/go/os/str.go b/libgo/go/os/str.go index 8dc9e47..e3606b6 100644 --- a/libgo/go/os/str.go +++ b/libgo/go/os/str.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build plan9 + package os func itoa(val int) string { // do it here rather than with fmt to avoid dependency diff --git a/libgo/go/os/sys_bsd.go b/libgo/go/os/sys_bsd.go index 188993b..b0d097a 100644 --- a/libgo/go/os/sys_bsd.go +++ b/libgo/go/os/sys_bsd.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd openbsd + // os code shared between *BSD systems including OS X (Darwin) // and FreeBSD. diff --git a/libgo/go/os/user/lookup_stubs.go b/libgo/go/os/user/lookup_stubs.go index 2f08f70..2d2de98 100644 --- a/libgo/go/os/user/lookup_stubs.go +++ b/libgo/go/os/user/lookup_stubs.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build openbsd plan9 windows + package user import ( diff --git a/libgo/go/os/user/lookup_unix.go b/libgo/go/os/user/lookup_unix.go index 18a9687..3336b52 100644 --- a/libgo/go/os/user/lookup_unix.go +++ b/libgo/go/os/user/lookup_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux + package user import ( diff --git a/libgo/go/path/filepath/match.go b/libgo/go/path/filepath/match.go index 7fcc214..0ccc87e 100644 --- a/libgo/go/path/filepath/match.go +++ b/libgo/go/path/filepath/match.go @@ -215,7 +215,7 @@ func getEsc(chunk string) (r int, nchunk string, err os.Error) { func Glob(pattern string) (matches []string, err os.Error) { if !hasMeta(pattern) { if _, err = os.Stat(pattern); err != nil { - return + return nil, nil } return []string{pattern}, nil } diff --git a/libgo/go/path/filepath/match_test.go b/libgo/go/path/filepath/match_test.go index 5fc7b9c..fa7aad9 100644 --- a/libgo/go/path/filepath/match_test.go +++ b/libgo/go/path/filepath/match_test.go @@ -125,6 +125,16 @@ func TestGlob(t *testing.T) { t.Errorf("Glob(%#q) = %#v want %v", tt.pattern, matches, tt.result) } } + for _, pattern := range []string{"no_match", "../*/no_match"} { + matches, err := Glob(pattern) + if err != nil { + t.Errorf("Glob error for %q: %s", pattern, err) + continue + } + if len(matches) != 0 { + t.Errorf("Glob(%#q) = %#v want []", pattern, matches) + } + } } func TestGlobError(t *testing.T) { diff --git a/libgo/go/path/filepath/path.go b/libgo/go/path/filepath/path.go index 3d5b915..afb8f10 100644 --- a/libgo/go/path/filepath/path.go +++ b/libgo/go/path/filepath/path.go @@ -41,9 +41,12 @@ func Clean(path string) string { vol := VolumeName(path) path = path[len(vol):] if path == "" { + if len(vol) > 1 && vol[1] != ':' { + // should be UNC + return FromSlash(vol) + } return vol + "." } - rooted := os.IsPathSeparator(path[0]) // Invariants: @@ -144,8 +147,9 @@ func SplitList(path string) []string { // If there is no Separator in path, Split returns an empty dir // and file set to path. func Split(path string) (dir, file string) { + vol := VolumeName(path) i := len(path) - 1 - for i >= 0 && !os.IsPathSeparator(path[i]) { + for i >= len(vol) && !os.IsPathSeparator(path[i]) { i-- } return path[:i+1], path[i+1:] @@ -254,38 +258,132 @@ func Abs(path string) (string, os.Error) { return Join(wd, path), nil } -// Visitor methods are invoked for corresponding file tree entries -// visited by Walk. The parameter path is the full path of f relative -// to root. -type Visitor interface { - VisitDir(path string, f *os.FileInfo) bool - VisitFile(path string, f *os.FileInfo) +// Rel returns a relative path that is lexically equivalent to targpath when +// joined to basepath with an intervening separator. That is, +// Join(basepath, Rel(basepath, targpath)) is equivalent to targpath itself. +// An error is returned if targpath can't be made relative to basepath or if +// knowing the current working directory would be necessary to compute it. +func Rel(basepath, targpath string) (string, os.Error) { + baseVol := VolumeName(basepath) + targVol := VolumeName(targpath) + base := Clean(basepath) + targ := Clean(targpath) + if targ == base { + return ".", nil + } + base = base[len(baseVol):] + targ = targ[len(targVol):] + if base == "." { + base = "" + } + // Can't use IsAbs - `\a` and `a` are both relative in Windows. + baseSlashed := len(base) > 0 && base[0] == Separator + targSlashed := len(targ) > 0 && targ[0] == Separator + if baseSlashed != targSlashed || baseVol != targVol { + return "", os.NewError("Rel: can't make " + targ + " relative to " + base) + } + // Position base[b0:bi] and targ[t0:ti] at the first differing elements. + bl := len(base) + tl := len(targ) + var b0, bi, t0, ti int + for { + for bi < bl && base[bi] != Separator { + bi++ + } + for ti < tl && targ[ti] != Separator { + ti++ + } + if targ[t0:ti] != base[b0:bi] { + break + } + if bi < bl { + bi++ + } + if ti < tl { + ti++ + } + b0 = bi + t0 = ti + } + if base[b0:bi] == ".." { + return "", os.NewError("Rel: can't make " + targ + " relative to " + base) + } + if b0 != bl { + // Base elements left. Must go up before going down. + seps := strings.Count(base[b0:bl], string(Separator)) + buf := make([]byte, 3+seps*3+tl-t0) + n := copy(buf, "..") + for i := 0; i < seps; i++ { + buf[n] = Separator + copy(buf[n+1:], "..") + n += 3 + } + if t0 != tl { + buf[n] = Separator + copy(buf[n+1:], targ[t0:]) + } + return string(buf), nil + } + return targ[t0:], nil } -func walk(path string, f *os.FileInfo, v Visitor, errors chan<- os.Error) { - if !f.IsDirectory() { - v.VisitFile(path, f) - return +// SkipDir is used as a return value from WalkFuncs to indicate that +// the directory named in the call is to be skipped. It is not returned +// as an error by any function. +var SkipDir = os.NewError("skip this directory") + +// WalkFunc is the type of the function called for each file or directory +// visited by Walk. If there was a problem walking to the file or directory +// named by path, the incoming error will describe the problem and the +// function can decide how to handle that error (and Walk will not descend +// into that directory). If an error is returned, processing stops. The +// sole exception is that if path is a directory and the function returns the +// special value SkipDir, the contents of the directory are skipped +// and processing continues as usual on the next file. +type WalkFunc func(path string, info *os.FileInfo, err os.Error) os.Error + +// walk recursively descends path, calling w. +func walk(path string, info *os.FileInfo, walkFn WalkFunc) os.Error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDirectory() && err == SkipDir { + return nil + } + return err } - if !v.VisitDir(path, f) { - return // skip directory entries + if !info.IsDirectory() { + return nil } list, err := readDir(path) if err != nil { - if errors != nil { - errors <- err + return walkFn(path, info, err) + } + + for _, fileInfo := range list { + if err = walk(Join(path, fileInfo.Name), fileInfo, walkFn); err != nil { + return err } } + return nil +} - for _, e := range list { - walk(Join(path, e.Name), e, v, errors) +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by walkFn. The files are walked in lexical +// order, which makes the output deterministic but means that for very +// large directories Walk can be inefficient. +func Walk(root string, walkFn WalkFunc) os.Error { + info, err := os.Lstat(root) + if err != nil { + return walkFn(root, nil, err) } + return walk(root, info, walkFn) } // readDir reads the directory named by dirname and returns -// a list of sorted directory entries. +// a sorted list of directory entries. // Copied from io/ioutil to avoid the circular import. func readDir(dirname string) ([]*os.FileInfo, os.Error) { f, err := os.Open(dirname) @@ -312,24 +410,6 @@ func (f fileInfoList) Len() int { return len(f) } func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name } func (f fileInfoList) Swap(i, j int) { f[i], f[j] = f[j], f[i] } -// Walk walks the file tree rooted at root, calling v.VisitDir or -// v.VisitFile for each directory or file in the tree, including root. -// If v.VisitDir returns false, Walk skips the directory's entries; -// otherwise it invokes itself for each directory entry in sorted order. -// An error reading a directory does not abort the Walk. -// If errors != nil, Walk sends each directory read error -// to the channel. Otherwise Walk discards the error. -func Walk(root string, v Visitor, errors chan<- os.Error) { - f, err := os.Lstat(root) - if err != nil { - if errors != nil { - errors <- err - } - return // can't progress - } - walk(root, f, v, errors) -} - // Base returns the last element of path. // Trailing path separators are removed before extracting the last element. // If the path is empty, Base returns ".". diff --git a/libgo/go/path/filepath/path_test.go b/libgo/go/path/filepath/path_test.go index e944bf4..f8e055b 100644 --- a/libgo/go/path/filepath/path_test.go +++ b/libgo/go/path/filepath/path_test.go @@ -72,15 +72,23 @@ var wincleantests = []PathTest{ {`c:\abc`, `c:\abc`}, {`c:abc\..\..\.\.\..\def`, `c:..\..\def`}, {`c:\abc\def\..\..`, `c:\`}, + {`c:\..\abc`, `c:\abc`}, {`c:..\abc`, `c:..\abc`}, {`\`, `\`}, {`/`, `\`}, + {`\\i\..\c$`, `\c$`}, + {`\\i\..\i\c$`, `\i\c$`}, + {`\\i\..\I\c$`, `\I\c$`}, + {`\\host\share\foo\..\bar`, `\\host\share\bar`}, + {`//host/share/foo/../baz`, `\\host\share\baz`}, + {`\\a\b\..\c`, `\\a\b\c`}, + {`\\a\b`, `\\a\b`}, } func TestClean(t *testing.T) { tests := cleantests if runtime.GOOS == "windows" { - for i, _ := range tests { + for i := range tests { tests[i].result = filepath.FromSlash(tests[i].result) } tests = append(tests, wincleantests...) @@ -145,9 +153,25 @@ var unixsplittests = []SplitTest{ {"/", "/", ""}, } +var winsplittests = []SplitTest{ + {`c:`, `c:`, ``}, + {`c:/`, `c:/`, ``}, + {`c:/foo`, `c:/`, `foo`}, + {`c:/foo/bar`, `c:/foo/`, `bar`}, + {`//host/share`, `//host/share`, ``}, + {`//host/share/`, `//host/share/`, ``}, + {`//host/share/foo`, `//host/share/`, `foo`}, + {`\\host\share`, `\\host\share`, ``}, + {`\\host\share\`, `\\host\share\`, ``}, + {`\\host\share\foo`, `\\host\share\`, `foo`}, +} + func TestSplit(t *testing.T) { var splittests []SplitTest splittests = unixsplittests + if runtime.GOOS == "windows" { + splittests = append(splittests, winsplittests...) + } for _, test := range splittests { if d, f := filepath.Split(test.path); d != test.dir || f != test.file { t.Errorf("Split(%q) = %q, %q, want %q, %q", test.path, d, f, test.dir, test.file) @@ -185,6 +209,8 @@ var winjointests = []JoinTest{ {[]string{`C:\Windows\`, ``}, `C:\Windows`}, {[]string{`C:\`, `Windows`}, `C:\Windows`}, {[]string{`C:`, `Windows`}, `C:\Windows`}, + {[]string{`\\host\share`, `foo`}, `\\host\share\foo`}, + {[]string{`//host/share`, `foo/bar`}, `\\host\share\foo\bar`}, } // join takes a []string and passes it to Join. @@ -279,9 +305,9 @@ func makeTree(t *testing.T) { func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } -func checkMarks(t *testing.T) { +func checkMarks(t *testing.T, report bool) { walkTree(tree, tree.name, func(path string, n *Node) { - if n.mark != 1 { + if n.mark != 1 && report { t.Errorf("node %s mark = %d; expected 1", path, n.mark) } n.mark = 0 @@ -289,44 +315,41 @@ func checkMarks(t *testing.T) { } // Assumes that each node name is unique. Good enough for a test. -func mark(name string) { - name = filepath.ToSlash(name) +// If clear is true, any incoming error is cleared before return. The errors +// are always accumulated, though. +func mark(path string, info *os.FileInfo, err os.Error, errors *[]os.Error, clear bool) os.Error { + if err != nil { + *errors = append(*errors, err) + if clear { + return nil + } + return err + } walkTree(tree, tree.name, func(path string, n *Node) { - if n.name == name { + if n.name == info.Name { n.mark++ } }) -} - -type TestVisitor struct{} - -func (v *TestVisitor) VisitDir(path string, f *os.FileInfo) bool { - mark(f.Name) - return true -} - -func (v *TestVisitor) VisitFile(path string, f *os.FileInfo) { - mark(f.Name) + return nil } func TestWalk(t *testing.T) { makeTree(t) - - // 1) ignore error handling, expect none - v := &TestVisitor{} - filepath.Walk(tree.name, v, nil) - checkMarks(t) - - // 2) handle errors, expect none - errors := make(chan os.Error, 64) - filepath.Walk(tree.name, v, errors) - select { - case err := <-errors: + errors := make([]os.Error, 0, 10) + clear := true + markFn := func(path string, info *os.FileInfo, err os.Error) os.Error { + return mark(path, info, err, &errors, clear) + } + // Expect no errors. + err := filepath.Walk(tree.name, markFn) + if err != nil { t.Errorf("no error expected, found: %s", err) - default: - // ok } - checkMarks(t) + if len(errors) != 0 { + t.Errorf("unexpected errors: %s", errors) + } + checkMarks(t, true) + errors = errors[0:0] // Test permission errors. Only possible if we're not root // and only on some file systems (AFS, FAT). To avoid errors during @@ -335,40 +358,50 @@ func TestWalk(t *testing.T) { // introduce 2 errors: chmod top-level directories to 0 os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) + + // 3) capture errors, expect two. // mark respective subtrees manually markTree(tree.entries[1]) markTree(tree.entries[3]) // correct double-marking of directory itself tree.entries[1].mark-- tree.entries[3].mark-- + err := filepath.Walk(tree.name, markFn) + if err != nil { + t.Errorf("expected no error return from Walk, %s", err) + } + if len(errors) != 2 { + t.Errorf("expected 2 errors, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, true) + errors = errors[0:0] - // 3) handle errors, expect two - errors = make(chan os.Error, 64) - os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) - filepath.Walk(tree.name, v, errors) - Loop: - for i := 1; i <= 2; i++ { - select { - case <-errors: - // ok - default: - t.Errorf("%d. error expected, none found", i) - break Loop - } + // 4) capture errors, stop after first error. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + clear = false // error will stop processing + err = filepath.Walk(tree.name, markFn) + if err == nil { + t.Errorf("expected error return from Walk") } - select { - case err := <-errors: - t.Errorf("only two errors expected, found 3rd: %v", err) - default: - // ok + if len(errors) != 1 { + t.Errorf("expected 1 error, got %d: %s", len(errors), errors) } // the inaccessible subtrees were marked manually - checkMarks(t) + checkMarks(t, false) + errors = errors[0:0] + + // restore permissions + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) } // cleanup - os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) - os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) if err := os.RemoveAll(tree.name); err != nil { t.Errorf("removeTree: %v", err) } @@ -421,6 +454,8 @@ var winisabstests = []IsAbsTest{ {`\`, false}, {`\Windows`, false}, {`c:a\b`, false}, + {`\\host\share\foo`, true}, + {`//host/share/foo/bar`, true}, } func TestIsAbs(t *testing.T) { @@ -540,10 +575,11 @@ var abstests = []string{ "pkg/../../AUTHORS", "Make.pkg", "pkg/Makefile", - - // Already absolute + ".", "$GOROOT/src/Make.pkg", "$GOROOT/src/../src/Make.pkg", + "$GOROOT/misc/cgo", + "$GOROOT", } func TestAbs(t *testing.T) { @@ -557,14 +593,15 @@ func TestAbs(t *testing.T) { os.Chdir(cwd) for _, path := range abstests { path = strings.Replace(path, "$GOROOT", goroot, -1) - abspath, err := filepath.Abs(path) - if err != nil { - t.Errorf("Abs(%q) error: %v", path, err) - } info, err := os.Stat(path) if err != nil { t.Errorf("%s: %s", path, err) } + + abspath, err := filepath.Abs(path) + if err != nil { + t.Errorf("Abs(%q) error: %v", path, err) + } absinfo, err := os.Stat(abspath) if err != nil || absinfo.Ino != info.Ino { t.Errorf("Abs(%q)=%q, not the same file", path, abspath) @@ -579,3 +616,114 @@ func TestAbs(t *testing.T) { } */ + +type RelTests struct { + root, path, want string +} + +var reltests = []RelTests{ + {"a/b", "a/b", "."}, + {"a/b/.", "a/b", "."}, + {"a/b", "a/b/.", "."}, + {"./a/b", "a/b", "."}, + {"a/b", "./a/b", "."}, + {"ab/cd", "ab/cde", "../cde"}, + {"ab/cd", "ab/c", "../c"}, + {"a/b", "a/b/c/d", "c/d"}, + {"a/b", "a/b/../c", "../c"}, + {"a/b/../c", "a/b", "../b"}, + {"a/b/c", "a/c/d", "../../c/d"}, + {"a/b", "c/d", "../../c/d"}, + {"../../a/b", "../../a/b/c/d", "c/d"}, + {"/a/b", "/a/b", "."}, + {"/a/b/.", "/a/b", "."}, + {"/a/b", "/a/b/.", "."}, + {"/ab/cd", "/ab/cde", "../cde"}, + {"/ab/cd", "/ab/c", "../c"}, + {"/a/b", "/a/b/c/d", "c/d"}, + {"/a/b", "/a/b/../c", "../c"}, + {"/a/b/../c", "/a/b", "../b"}, + {"/a/b/c", "/a/c/d", "../../c/d"}, + {"/a/b", "/c/d", "../../c/d"}, + {"/../../a/b", "/../../a/b/c/d", "c/d"}, + {".", "a/b", "a/b"}, + {".", "..", ".."}, + + // can't do purely lexically + {"..", ".", "err"}, + {"..", "a", "err"}, + {"../..", "..", "err"}, + {"a", "/a", "err"}, + {"/a", "a", "err"}, +} + +var winreltests = []RelTests{ + {`C:a\b\c`, `C:a/b/d`, `..\d`}, + {`C:\`, `D:\`, `err`}, + {`C:`, `D:`, `err`}, +} + +func TestRel(t *testing.T) { + tests := append([]RelTests{}, reltests...) + if runtime.GOOS == "windows" { + for i := range tests { + tests[i].want = filepath.FromSlash(tests[i].want) + } + tests = append(tests, winreltests...) + } + for _, test := range tests { + got, err := filepath.Rel(test.root, test.path) + if test.want == "err" { + if err == nil { + t.Errorf("Rel(%q, %q)=%q, want error", test.root, test.path, got) + } + continue + } + if err != nil { + t.Errorf("Rel(%q, %q): want %q, got error: %s", test.root, test.path, test.want, err) + } + if got != test.want { + t.Errorf("Rel(%q, %q)=%q, want %q", test.root, test.path, got, test.want) + } + } +} + +type VolumeNameTest struct { + path string + vol string +} + +var volumenametests = []VolumeNameTest{ + {`c:/foo/bar`, `c:`}, + {`c:`, `c:`}, + {``, ``}, + {`\\\host`, ``}, + {`\\\host\`, ``}, + {`\\\host\share`, ``}, + {`\\\host\\share`, ``}, + {`\\host`, ``}, + {`//host`, ``}, + {`\\host\`, ``}, + {`//host/`, ``}, + {`\\host\share`, `\\host\share`}, + {`//host/share`, `//host/share`}, + {`\\host\share\`, `\\host\share`}, + {`//host/share/`, `//host/share`}, + {`\\host\share\foo`, `\\host\share`}, + {`//host/share/foo`, `//host/share`}, + {`\\host\share\\foo\\\bar\\\\baz`, `\\host\share`}, + {`//host/share//foo///bar////baz`, `//host/share`}, + {`\\host\share\foo\..\bar`, `\\host\share`}, + {`//host/share/foo/../bar`, `//host/share`}, +} + +func TestVolumeName(t *testing.T) { + if runtime.GOOS != "windows" { + return + } + for _, v := range volumenametests { + if vol := filepath.VolumeName(v.path); vol != v.vol { + t.Errorf("VolumeName(%q)=%q, want %q", v.path, vol, v.vol) + } + } +} diff --git a/libgo/go/path/filepath/path_unix.go b/libgo/go/path/filepath/path_unix.go index b2a4151..daf0eb2 100644 --- a/libgo/go/path/filepath/path_unix.go +++ b/libgo/go/path/filepath/path_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + package filepath import "strings" diff --git a/libgo/go/path/filepath/path_windows.go b/libgo/go/path/filepath/path_windows.go index 2535697..9692fd9 100644 --- a/libgo/go/path/filepath/path_windows.go +++ b/libgo/go/path/filepath/path_windows.go @@ -4,7 +4,13 @@ package filepath -import "strings" +import ( + "strings" +) + +func isSlash(c uint8) bool { + return c == '\\' || c == '/' +} // IsAbs returns true if the path is absolute. func IsAbs(path string) (b bool) { @@ -16,11 +22,12 @@ func IsAbs(path string) (b bool) { if path == "" { return false } - return path[0] == '/' || path[0] == '\\' + return isSlash(path[0]) } // VolumeName returns leading volume name. // Given "C:\foo\bar" it returns "C:" under windows. +// Given "\\host\share\foo" it returns "\\host\share". // On other platforms it returns "". func VolumeName(path string) (v string) { if len(path) < 2 { @@ -33,6 +40,30 @@ func VolumeName(path string) (v string) { 'A' <= c && c <= 'Z') { return path[:2] } + // is it UNC + if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && + !isSlash(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if isSlash(path[n]) { + n++ + // third, following something characters. its share name. + if !isSlash(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if isSlash(path[n]) { + break + } + } + return path[:n] + } + break + } + } + } return "" } diff --git a/libgo/go/rand/rand_test.go b/libgo/go/rand/rand_test.go index 3ebc114..f997ff5 100644 --- a/libgo/go/rand/rand_test.go +++ b/libgo/go/rand/rand_test.go @@ -30,11 +30,11 @@ func max(a, b float64) float64 { } func nearEqual(a, b, closeEnough, maxError float64) bool { - absDiff := math.Fabs(a - b) + absDiff := math.Abs(a - b) if absDiff < closeEnough { // Necessary when one value is zero and one value is close to zero. return true } - return absDiff/max(math.Fabs(a), math.Fabs(b)) < maxError + return absDiff/max(math.Abs(a), math.Abs(b)) < maxError } var testSeeds = []int64{1, 1754801282, 1698661970, 1550503961} diff --git a/libgo/go/reflect/all_test.go b/libgo/go/reflect/all_test.go index 8fa9d3d..696bd56 100644 --- a/libgo/go/reflect/all_test.go +++ b/libgo/go/reflect/all_test.go @@ -6,7 +6,7 @@ package reflect_test import ( "bytes" - "container/vector" + "encoding/base64" "fmt" "io" "os" @@ -877,19 +877,20 @@ func TestMap(t *testing.T) { t.Errorf("Len = %d, want %d", n, len(m)) } keys := mv.MapKeys() - i := 0 newmap := MakeMap(mv.Type()) for k, v := range m { // Check that returned Keys match keys in range. - // These aren't required to be in the same order, - // but they are in this implementation, which makes - // the test easier. - if i >= len(keys) { - t.Errorf("Missing key #%d %q", i, k) - } else if kv := keys[i]; kv.String() != k { - t.Errorf("Keys[%q] = %d, want %d", i, kv.Int(), k) + // These aren't required to be in the same order. + seen := false + for _, kv := range keys { + if kv.String() == k { + seen = true + break + } + } + if !seen { + t.Errorf("Missing key %q", k) } - i++ // Check that value lookup is correct. vv := mv.MapIndex(ValueOf(k)) @@ -1322,8 +1323,8 @@ func TestFieldByName(t *testing.T) { } func TestImportPath(t *testing.T) { - if path := TypeOf(vector.Vector{}).PkgPath(); path != "libgo_container.vector" { - t.Errorf("TypeOf(vector.Vector{}).PkgPath() = %q, want \"libgo_container.vector\"", path) + if path := TypeOf(&base64.Encoding{}).Elem().PkgPath(); path != "libgo_encoding.base64" { + t.Errorf(`TypeOf(&base64.Encoding{}).Elem().PkgPath() = %q, want "libgo_encoding.base64"`, path) } } @@ -1567,6 +1568,31 @@ func TestTagGet(t *testing.T) { } } +func TestBytes(t *testing.T) { + type B []byte + x := B{1, 2, 3, 4} + y := ValueOf(x).Bytes() + if !bytes.Equal(x, y) { + t.Fatalf("ValueOf(%v).Bytes() = %v", x, y) + } + if &x[0] != &y[0] { + t.Errorf("ValueOf(%p).Bytes() = %p", &x[0], &y[0]) + } +} + +func TestSetBytes(t *testing.T) { + type B []byte + var x B + y := []byte{1, 2, 3, 4} + ValueOf(&x).Elem().SetBytes(y) + if !bytes.Equal(x, y) { + t.Fatalf("ValueOf(%v).Bytes() = %v", x, y) + } + if &x[0] != &y[0] { + t.Errorf("ValueOf(%p).Bytes() = %p", &x[0], &y[0]) + } +} + type Private struct { x int y **int diff --git a/libgo/go/reflect/type.go b/libgo/go/reflect/type.go index 6d0f454..9be7f83 100644 --- a/libgo/go/reflect/type.go +++ b/libgo/go/reflect/type.go @@ -68,7 +68,7 @@ type Type interface { Name() string // PkgPath returns the type's package path. - // The package path is a full package import path like "container/vector". + // The package path is a full package import path like "encoding/base64". // PkgPath returns an empty string for unnamed types. PkgPath() string @@ -78,7 +78,7 @@ type Type interface { // String returns a string representation of the type. // The string representation may use shortened package names - // (e.g., vector instead of "container/vector") and is not + // (e.g., base64 instead of "encoding/base64") and is not // guaranteed to be unique among types. To test for equality, // compare the Types directly. String() string @@ -717,7 +717,7 @@ type StructTag string // Get returns the value associated with key in the tag string. // If there is no such key in the tag, Get returns the empty string. // If the tag does not have the conventional format, the value -// returned by Get is unspecified, +// returned by Get is unspecified. func (tag StructTag) Get(key string) string { for tag != "" { // skip leading space @@ -877,7 +877,7 @@ L: fd = inf } - mark[t] = false, false + delete(mark, t) return } diff --git a/libgo/go/reflect/value.go b/libgo/go/reflect/value.go index 0d916e8..4a21075 100644 --- a/libgo/go/reflect/value.go +++ b/libgo/go/reflect/value.go @@ -323,8 +323,31 @@ func packValue(flag uint32, typ *runtime.Type, word iword) Value { return Value{Internal: *(*interface{})(unsafe.Pointer(&eface))} } +var dummy struct { + b bool + x interface{} +} + +// Dummy annotation marking that the value x escapes, +// for use in cases where the reflect code is so clever that +// the compiler cannot follow. +func escapes(x interface{}) { + if dummy.b { + dummy.x = x + } +} + // valueFromAddr returns a Value using the given type and address. func valueFromAddr(flag uint32, typ Type, addr unsafe.Pointer) Value { + // TODO(rsc): Eliminate this terrible hack. + // The escape analysis knows that addr is a pointer + // but it doesn't see addr get passed to anything + // that keeps it. packValue keeps it, but packValue + // takes a uintptr (iword(addr)), and integers (non-pointers) + // are assumed not to matter. The escapes function works + // because return values always escape (for now). + escapes(addr) + if flag&flagAddr != 0 { // Addressable, so the internal value is // an interface containing a pointer to the real value. @@ -399,6 +422,18 @@ func (v Value) Bool() bool { return *(*bool)(unsafe.Pointer(iv.addr)) } +// Bytes returns v's underlying value. +// It panics if v's underlying value is not a slice of bytes. +func (v Value) Bytes() []byte { + iv := v.internal() + iv.mustBe(Slice) + typ := iv.typ.toType() + if typ.Elem().Kind() != Uint8 { + panic("reflect.Value.Bytes of non-byte slice") + } + return *(*[]byte)(iv.addr) +} + // CanAddr returns true if the value's address can be obtained with Addr. // Such values are called addressable. A value is addressable if it is // an element of a slice, an element of an addressable array, @@ -1221,6 +1256,19 @@ func (v Value) SetBool(x bool) { *(*bool)(iv.addr) = x } +// SetBytes sets v's underlying value. +// It panics if v's underlying value is not a slice of bytes. +func (v Value) SetBytes(x []byte) { + iv := v.internal() + iv.mustBeAssignable() + iv.mustBe(Slice) + typ := iv.typ.toType() + if typ.Elem().Kind() != Uint8 { + panic("reflect.Value.SetBytes of non-byte slice") + } + *(*[]byte)(iv.addr) = x +} + // SetComplex sets v's underlying value to x. // It panics if v's Kind is not Complex64 or Complex128, or if CanSet() is false. func (v Value) SetComplex(x complex128) { @@ -1374,11 +1422,17 @@ func (v Value) Slice(beg, end int) Value { typ = iv.typ.toType() base = (*SliceHeader)(iv.addr).Data } - s := new(SliceHeader) + + // Declare slice so that gc can see the base pointer in it. + var x []byte + + // Reinterpret as *SliceHeader to edit. + s := (*SliceHeader)(unsafe.Pointer(&x)) s.Data = base + uintptr(beg)*typ.Elem().Size() s.Len = end - beg - s.Cap = cap - beg - return valueFromAddr(iv.flag&flagRO, typ, unsafe.Pointer(s)) + s.Cap = end - beg + + return valueFromAddr(iv.flag&flagRO, typ, unsafe.Pointer(&x)) } // String returns the string v's underlying value, as a string. @@ -1393,6 +1447,8 @@ func (v Value) String() string { case String: return *(*string)(iv.addr) } + // If you call String on a reflect.Value of other type, it's better to + // print something than to panic. Useful in debugging. return "<" + iv.typ.String() + " Value>" } @@ -1599,12 +1655,17 @@ func MakeSlice(typ Type, len, cap int) Value { if typ.Kind() != Slice { panic("reflect: MakeSlice of non-slice type") } - s := &SliceHeader{ - Data: uintptr(unsafe.NewArray(typ.Elem(), cap)), - Len: len, - Cap: cap, - } - return valueFromAddr(0, typ, unsafe.Pointer(s)) + + // Declare slice so that gc can see the base pointer in it. + var x []byte + + // Reinterpret as *SliceHeader to edit. + s := (*SliceHeader)(unsafe.Pointer(&x)) + s.Data = uintptr(unsafe.NewArray(typ.Elem(), cap)) + s.Len = len + s.Cap = cap + + return valueFromAddr(0, typ, unsafe.Pointer(&x)) } // MakeChan creates a new channel with the specified type and buffer size. @@ -1647,6 +1708,14 @@ func ValueOf(i interface{}) Value { if i == nil { return Value{} } + + // TODO(rsc): Eliminate this terrible hack. + // In the call to packValue, eface.typ doesn't escape, + // and eface.word is an integer. So it looks like + // i (= eface) doesn't escape. But really it does, + // because eface.word is actually a pointer. + escapes(i) + // For an interface value with the noAddr bit set, // the representation is identical to an empty interface. eface := *(*emptyInterface)(unsafe.Pointer(&i)) diff --git a/libgo/go/regexp/all_test.go b/libgo/go/regexp/all_test.go index 71edc4d..77f32ca 100644 --- a/libgo/go/regexp/all_test.go +++ b/libgo/go/regexp/all_test.go @@ -24,13 +24,13 @@ var good_re = []string{ `[a-z]`, `[a-abc-c\-\]\[]`, `[a-z]+`, - `[]`, `[abc]`, `[^1234]`, `[^\n]`, `\!\\`, } +/* type stringError struct { re string err os.Error @@ -51,6 +51,7 @@ var bad_re = []stringError{ {`a??`, ErrBadClosure}, {`\x`, ErrBadBackslash}, } +*/ func compileTest(t *testing.T, expr string, error os.Error) *Regexp { re, err := Compile(expr) @@ -66,11 +67,13 @@ func TestGoodCompile(t *testing.T) { } } +/* func TestBadCompile(t *testing.T) { for i := 0; i < len(bad_re); i++ { compileTest(t, bad_re[i].re, bad_re[i].err) } } +*/ func matchTest(t *testing.T, test *FindTest) { re := compileTest(t, test.pat, nil) @@ -240,7 +243,7 @@ var metaTests = []MetaTest{ {`foo`, `foo`, `foo`, true}, {`foo\.\$`, `foo\\\.\\\$`, `foo.$`, true}, // has meta but no operator {`foo.\$`, `foo\.\\\$`, `foo`, false}, // has escaped operators and real operators - {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[{\]}\\\|,<\.>/\?~`, `!@#`, false}, + {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[\{\]\}\\\|,<\.>/\?~`, `!@#`, false}, } func TestQuoteMeta(t *testing.T) { diff --git a/libgo/go/exp/regexp/exec.go b/libgo/go/regexp/exec.go index 0670bb9..3b0e388 100644 --- a/libgo/go/exp/regexp/exec.go +++ b/libgo/go/regexp/exec.go @@ -1,6 +1,6 @@ package regexp -import "exp/regexp/syntax" +import "regexp/syntax" // A queue is a 'sparse array' holding pending threads of execution. // See http://research.swtch.com/2008/03/using-uninitialized-memory-for-fun-and.html @@ -50,6 +50,13 @@ func progMachine(p *syntax.Prog) *machine { return m } +func (m *machine) init(ncap int) { + for _, t := range m.pool { + t.cap = t.cap[:ncap] + } + m.matchcap = m.matchcap[:ncap] +} + // alloc allocates a new thread with the given instruction. // It uses the free pool if possible. func (m *machine) alloc(i *syntax.Inst) *thread { @@ -59,9 +66,8 @@ func (m *machine) alloc(i *syntax.Inst) *thread { m.pool = m.pool[:n-1] } else { t = new(thread) - t.cap = make([]int, cap(m.matchcap)) + t.cap = make([]int, len(m.matchcap), cap(m.matchcap)) } - t.cap = t.cap[:len(m.matchcap)] t.inst = i return t } @@ -90,23 +96,12 @@ func (m *machine) match(i input, pos int) bool { if rune != endOfText { rune1, width1 = i.step(pos + width) } - // TODO: Let caller specify the initial flag setting. - // For now assume pos == 0 is beginning of text and - // pos != 0 is not even beginning of line. - // TODO: Word boundary. var flag syntax.EmptyOp if pos == 0 { - flag = syntax.EmptyBeginText | syntax.EmptyBeginLine - } - - // Update flag using lookahead rune. - if rune1 == '\n' { - flag |= syntax.EmptyEndLine - } - if rune1 == endOfText { - flag |= syntax.EmptyEndText + flag = syntax.EmptyOpContext(-1, rune) + } else { + flag = i.context(pos) } - for { if len(runq.dense) == 0 { if startCond&syntax.EmptyBeginText != 0 && pos != 0 { @@ -132,23 +127,18 @@ func (m *machine) match(i input, pos int) bool { if len(m.matchcap) > 0 { m.matchcap[0] = pos } - m.add(runq, uint32(m.p.Start), pos, m.matchcap, flag) - } - // TODO: word boundary - flag = 0 - if rune == '\n' { - flag |= syntax.EmptyBeginLine - } - if rune1 == '\n' { - flag |= syntax.EmptyEndLine - } - if rune1 == endOfText { - flag |= syntax.EmptyEndText + m.add(runq, uint32(m.p.Start), pos, m.matchcap, flag, nil) } + flag = syntax.EmptyOpContext(rune, rune1) m.step(runq, nextq, pos, pos+width, rune, flag) if width == 0 { break } + if len(m.matchcap) == 0 && m.matched { + // Found a match and not paying attention + // to where it is, so any match will do. + break + } pos += width rune, width = rune1, width1 if rune != endOfText { @@ -164,7 +154,8 @@ func (m *machine) match(i input, pos int) bool { func (m *machine) clear(q *queue) { for _, d := range q.dense { if d.t != nil { - m.free(d.t) + // m.free(d.t) + m.pool = append(m.pool, d.t) } } q.dense = q.dense[:0] @@ -176,44 +167,57 @@ func (m *machine) clear(q *queue) { // which starts at position pos and ends at nextPos. // nextCond gives the setting for the empty-width flags after c. func (m *machine) step(runq, nextq *queue, pos, nextPos, c int, nextCond syntax.EmptyOp) { + longest := m.re.longest for j := 0; j < len(runq.dense); j++ { d := &runq.dense[j] t := d.t if t == nil { continue } - /* - * If we support leftmost-longest matching: - if longest && matched && match[0] < t.cap[0] { - m.free(t) - continue - } - */ - + if longest && m.matched && len(t.cap) > 0 && m.matchcap[0] < t.cap[0] { + // m.free(t) + m.pool = append(m.pool, t) + continue + } i := t.inst + add := false switch i.Op { default: panic("bad inst") case syntax.InstMatch: - if len(t.cap) > 0 { + if len(t.cap) > 0 && (!longest || !m.matched || m.matchcap[1] < pos) { t.cap[1] = pos copy(m.matchcap, t.cap) } - m.matched = true - for _, d := range runq.dense[j+1:] { - if d.t != nil { - m.free(d.t) + if !longest { + // First-match mode: cut off all lower-priority threads. + for _, d := range runq.dense[j+1:] { + if d.t != nil { + // m.free(d.t) + m.pool = append(m.pool, d.t) + } } + runq.dense = runq.dense[:0] } - runq.dense = runq.dense[:0] + m.matched = true case syntax.InstRune: - if i.MatchRune(c) { - m.add(nextq, i.Out, nextPos, t.cap, nextCond) - } + add = i.MatchRune(c) + case syntax.InstRune1: + add = c == i.Rune[0] + case syntax.InstRuneAny: + add = true + case syntax.InstRuneAnyNotNL: + add = c != '\n' + } + if add { + t = m.add(nextq, i.Out, nextPos, t.cap, nextCond, t) + } + if t != nil { + // m.free(t) + m.pool = append(m.pool, t) } - m.free(t) } runq.dense = runq.dense[:0] } @@ -222,12 +226,12 @@ func (m *machine) step(runq, nextq *queue, pos, nextPos, c int, nextCond syntax. // It also recursively adds an entry for all instructions reachable from pc by following // empty-width conditions satisfied by cond. pos gives the current position // in the input. -func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.EmptyOp) { +func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.EmptyOp, t *thread) *thread { if pc == 0 { - return + return t } if j := q.sparse[pc]; j < uint32(len(q.dense)) && q.dense[j].pc == pc { - return + return t } j := len(q.dense) @@ -244,30 +248,36 @@ func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.Empty case syntax.InstFail: // nothing case syntax.InstAlt, syntax.InstAltMatch: - m.add(q, i.Out, pos, cap, cond) - m.add(q, i.Arg, pos, cap, cond) + t = m.add(q, i.Out, pos, cap, cond, t) + t = m.add(q, i.Arg, pos, cap, cond, t) case syntax.InstEmptyWidth: if syntax.EmptyOp(i.Arg)&^cond == 0 { - m.add(q, i.Out, pos, cap, cond) + t = m.add(q, i.Out, pos, cap, cond, t) } case syntax.InstNop: - m.add(q, i.Out, pos, cap, cond) + t = m.add(q, i.Out, pos, cap, cond, t) case syntax.InstCapture: if int(i.Arg) < len(cap) { opos := cap[i.Arg] cap[i.Arg] = pos - m.add(q, i.Out, pos, cap, cond) + m.add(q, i.Out, pos, cap, cond, nil) cap[i.Arg] = opos } else { - m.add(q, i.Out, pos, cap, cond) + t = m.add(q, i.Out, pos, cap, cond, t) } - case syntax.InstMatch, syntax.InstRune: - t := m.alloc(i) - if len(t.cap) > 0 { + case syntax.InstMatch, syntax.InstRune, syntax.InstRune1, syntax.InstRuneAny, syntax.InstRuneAnyNotNL: + if t == nil { + t = m.alloc(i) + } else { + t.inst = i + } + if len(cap) > 0 && &t.cap[0] != &cap[0] { copy(t.cap, cap) } d.t = t + t = nil } + return t } // empty is a non-nil 0-element slice, @@ -279,7 +289,7 @@ var empty = make([]int, 0) // the position of its subexpressions. func (re *Regexp) doExecute(i input, pos int, ncap int) []int { m := re.get() - m.matchcap = m.matchcap[:ncap] + m.init(ncap) if !m.match(i, pos) { re.put(m) return nil diff --git a/libgo/go/regexp/exec_test.go b/libgo/go/regexp/exec_test.go new file mode 100644 index 0000000..905fd4e --- /dev/null +++ b/libgo/go/regexp/exec_test.go @@ -0,0 +1,734 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regexp + +import ( + "bufio" + "compress/bzip2" + "fmt" + "io" + old "old/regexp" + "os" + "path/filepath" + "rand" + "regexp/syntax" + "strconv" + "strings" + "testing" + "utf8" +) + +// TestRE2 tests this package's regexp API against test cases +// considered during RE2's exhaustive tests, which run all possible +// regexps over a given set of atoms and operators, up to a given +// complexity, over all possible strings over a given alphabet, +// up to a given size. Rather than try to link with RE2, we read a +// log file containing the test cases and the expected matches. +// The log file, re2.txt, is generated by running 'make exhaustive-log' +// in the open source RE2 distribution. http://code.google.com/p/re2/ +// +// The test file format is a sequence of stanzas like: +// +// strings +// "abc" +// "123x" +// regexps +// "[a-z]+" +// 0-3;0-3 +// -;- +// "([0-9])([0-9])([0-9])" +// -;- +// -;0-3 0-1 1-2 2-3 +// +// The stanza begins by defining a set of strings, quoted +// using Go double-quote syntax, one per line. Then the +// regexps section gives a sequence of regexps to run on +// the strings. In the block that follows a regexp, each line +// gives the semicolon-separated match results of running +// the regexp on the corresponding string. +// Each match result is either a single -, meaning no match, or a +// space-separated sequence of pairs giving the match and +// submatch indices. An unmatched subexpression formats +// its pair as a single - (not illustrated above). For now +// each regexp run produces two match results, one for a +// ``full match'' that restricts the regexp to matching the entire +// string or nothing, and one for a ``partial match'' that gives +// the leftmost first match found in the string. +// +// Lines beginning with # are comments. Lines beginning with +// a capital letter are test names printed during RE2's test suite +// and are echoed into t but otherwise ignored. +// +// At time of writing, re2.txt is 32 MB but compresses to 760 kB, +// so we store re2.txt.gz in the repository and decompress it on the fly. +// +func TestRE2Search(t *testing.T) { + testRE2(t, "testdata/re2-search.txt") +} + +func TestRE2Exhaustive(t *testing.T) { + if testing.Short() { + t.Log("skipping TestRE2Exhaustive during short test") + return + } + testRE2(t, "testdata/re2-exhaustive.txt.bz2") +} + +func testRE2(t *testing.T, file string) { + f, err := os.Open(file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var txt io.Reader + if strings.HasSuffix(file, ".bz2") { + z := bzip2.NewReader(f) + txt = z + file = file[:len(file)-len(".bz2")] // for error messages + } else { + txt = f + } + lineno := 0 + r := bufio.NewReader(txt) + var ( + str []string + input []string + inStrings bool + re *Regexp + refull *Regexp + nfail int + ncase int + ) + for { + line, err := r.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + t.Fatalf("%s:%d: %v", file, lineno, err) + } + line = line[:len(line)-1] // chop \n + lineno++ + switch { + case line == "": + t.Fatalf("%s:%d: unexpected blank line", file, lineno) + case line[0] == '#': + continue + case 'A' <= line[0] && line[0] <= 'Z': + // Test name. + t.Logf("%s\n", line) + continue + case line == "strings": + str = str[:0] + inStrings = true + case line == "regexps": + inStrings = false + case line[0] == '"': + q, err := strconv.Unquote(line) + if err != nil { + // Fatal because we'll get out of sync. + t.Fatalf("%s:%d: unquote %s: %v", file, lineno, line, err) + } + if inStrings { + str = append(str, q) + continue + } + // Is a regexp. + if len(input) != 0 { + t.Fatalf("%s:%d: out of sync: have %d strings left before %#q", file, lineno, len(input), q) + } + re, err = tryCompile(q) + if err != nil { + if err.String() == "error parsing regexp: invalid escape sequence: `\\C`" { + // We don't and likely never will support \C; keep going. + continue + } + t.Errorf("%s:%d: compile %#q: %v", file, lineno, q, err) + if nfail++; nfail >= 100 { + t.Fatalf("stopping after %d errors", nfail) + } + continue + } + full := `\A(?:` + q + `)\z` + refull, err = tryCompile(full) + if err != nil { + // Fatal because q worked, so this should always work. + t.Fatalf("%s:%d: compile full %#q: %v", file, lineno, full, err) + } + input = str + case line[0] == '-' || '0' <= line[0] && line[0] <= '9': + // A sequence of match results. + ncase++ + if re == nil { + // Failed to compile: skip results. + continue + } + if len(input) == 0 { + t.Fatalf("%s:%d: out of sync: no input remaining", file, lineno) + } + var text string + text, input = input[0], input[1:] + if !isSingleBytes(text) && strings.Contains(re.String(), `\B`) { + // RE2's \B considers every byte position, + // so it sees 'not word boundary' in the + // middle of UTF-8 sequences. This package + // only considers the positions between runes, + // so it disagrees. Skip those cases. + continue + } + res := strings.Split(line, ";") + if len(res) != len(run) { + t.Fatalf("%s:%d: have %d test results, want %d", file, lineno, len(res), len(run)) + } + for i := range res { + have, suffix := run[i](re, refull, text) + want := parseResult(t, file, lineno, res[i]) + if !same(have, want) { + t.Errorf("%s:%d: %#q%s.FindSubmatchIndex(%#q) = %v, want %v", file, lineno, re, suffix, text, have, want) + if nfail++; nfail >= 100 { + t.Fatalf("stopping after %d errors", nfail) + } + continue + } + b, suffix := match[i](re, refull, text) + if b != (want != nil) { + t.Errorf("%s:%d: %#q%s.MatchString(%#q) = %v, want %v", file, lineno, re, suffix, text, b, !b) + if nfail++; nfail >= 100 { + t.Fatalf("stopping after %d errors", nfail) + } + continue + } + } + + default: + t.Fatalf("%s:%d: out of sync: %s\n", file, lineno, line) + } + } + if len(input) != 0 { + t.Fatalf("%s:%d: out of sync: have %d strings left at EOF", file, lineno, len(input)) + } + t.Logf("%d cases tested", ncase) +} + +var run = []func(*Regexp, *Regexp, string) ([]int, string){ + runFull, + runPartial, + runFullLongest, + runPartialLongest, +} + +func runFull(re, refull *Regexp, text string) ([]int, string) { + refull.longest = false + return refull.FindStringSubmatchIndex(text), "[full]" +} + +func runPartial(re, refull *Regexp, text string) ([]int, string) { + re.longest = false + return re.FindStringSubmatchIndex(text), "" +} + +func runFullLongest(re, refull *Regexp, text string) ([]int, string) { + refull.longest = true + return refull.FindStringSubmatchIndex(text), "[full,longest]" +} + +func runPartialLongest(re, refull *Regexp, text string) ([]int, string) { + re.longest = true + return re.FindStringSubmatchIndex(text), "[longest]" +} + +var match = []func(*Regexp, *Regexp, string) (bool, string){ + matchFull, + matchPartial, + matchFullLongest, + matchPartialLongest, +} + +func matchFull(re, refull *Regexp, text string) (bool, string) { + refull.longest = false + return refull.MatchString(text), "[full]" +} + +func matchPartial(re, refull *Regexp, text string) (bool, string) { + re.longest = false + return re.MatchString(text), "" +} + +func matchFullLongest(re, refull *Regexp, text string) (bool, string) { + refull.longest = true + return refull.MatchString(text), "[full,longest]" +} + +func matchPartialLongest(re, refull *Regexp, text string) (bool, string) { + re.longest = true + return re.MatchString(text), "[longest]" +} + +func isSingleBytes(s string) bool { + for _, c := range s { + if c >= utf8.RuneSelf { + return false + } + } + return true +} + +func tryCompile(s string) (re *Regexp, err os.Error) { + // Protect against panic during Compile. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + return Compile(s) +} + +func parseResult(t *testing.T, file string, lineno int, res string) []int { + // A single - indicates no match. + if res == "-" { + return nil + } + // Otherwise, a space-separated list of pairs. + n := 1 + for j := 0; j < len(res); j++ { + if res[j] == ' ' { + n++ + } + } + out := make([]int, 2*n) + i := 0 + n = 0 + for j := 0; j <= len(res); j++ { + if j == len(res) || res[j] == ' ' { + // Process a single pair. - means no submatch. + pair := res[i:j] + if pair == "-" { + out[n] = -1 + out[n+1] = -1 + } else { + k := strings.Index(pair, "-") + if k < 0 { + t.Fatalf("%s:%d: invalid pair %s", file, lineno, pair) + } + lo, err1 := strconv.Atoi(pair[:k]) + hi, err2 := strconv.Atoi(pair[k+1:]) + if err1 != nil || err2 != nil || lo > hi { + t.Fatalf("%s:%d: invalid pair %s", file, lineno, pair) + } + out[n] = lo + out[n+1] = hi + } + n += 2 + i = j + 1 + } + } + return out +} + +func same(x, y []int) bool { + if len(x) != len(y) { + return false + } + for i, xi := range x { + if xi != y[i] { + return false + } + } + return true +} + +// TestFowler runs this package's regexp API against the +// POSIX regular expression tests collected by Glenn Fowler +// at http://www2.research.att.com/~gsf/testregex/. +func TestFowler(t *testing.T) { + files, err := filepath.Glob("testdata/*.dat") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + t.Log(file) + testFowler(t, file) + } +} + +var notab = MustCompilePOSIX(`[^\t]+`) + +func testFowler(t *testing.T, file string) { + f, err := os.Open(file) + if err != nil { + t.Error(err) + return + } + defer f.Close() + b := bufio.NewReader(f) + lineno := 0 + lastRegexp := "" +Reading: + for { + lineno++ + line, err := b.ReadString('\n') + if err != nil { + if err != os.EOF { + t.Errorf("%s:%d: %v", file, lineno, err) + } + break Reading + } + + // http://www2.research.att.com/~gsf/man/man1/testregex.html + // + // INPUT FORMAT + // Input lines may be blank, a comment beginning with #, or a test + // specification. A specification is five fields separated by one + // or more tabs. NULL denotes the empty string and NIL denotes the + // 0 pointer. + if line[0] == '#' || line[0] == '\n' { + continue Reading + } + line = line[:len(line)-1] + field := notab.FindAllString(line, -1) + for i, f := range field { + if f == "NULL" { + field[i] = "" + } + if f == "NIL" { + t.Logf("%s:%d: skip: %s", file, lineno, line) + continue Reading + } + } + if len(field) == 0 { + continue Reading + } + + // Field 1: the regex(3) flags to apply, one character per REG_feature + // flag. The test is skipped if REG_feature is not supported by the + // implementation. If the first character is not [BEASKLP] then the + // specification is a global control line. One or more of [BEASKLP] may be + // specified; the test will be repeated for each mode. + // + // B basic BRE (grep, ed, sed) + // E REG_EXTENDED ERE (egrep) + // A REG_AUGMENTED ARE (egrep with negation) + // S REG_SHELL SRE (sh glob) + // K REG_SHELL|REG_AUGMENTED KRE (ksh glob) + // L REG_LITERAL LRE (fgrep) + // + // a REG_LEFT|REG_RIGHT implicit ^...$ + // b REG_NOTBOL lhs does not match ^ + // c REG_COMMENT ignore space and #...\n + // d REG_SHELL_DOT explicit leading . match + // e REG_NOTEOL rhs does not match $ + // f REG_MULTIPLE multiple \n separated patterns + // g FNM_LEADING_DIR testfnmatch only -- match until / + // h REG_MULTIREF multiple digit backref + // i REG_ICASE ignore case + // j REG_SPAN . matches \n + // k REG_ESCAPE \ to ecape [...] delimiter + // l REG_LEFT implicit ^... + // m REG_MINIMAL minimal match + // n REG_NEWLINE explicit \n match + // o REG_ENCLOSED (|&) magic inside [@|&](...) + // p REG_SHELL_PATH explicit / match + // q REG_DELIMITED delimited pattern + // r REG_RIGHT implicit ...$ + // s REG_SHELL_ESCAPED \ not special + // t REG_MUSTDELIM all delimiters must be specified + // u standard unspecified behavior -- errors not counted + // v REG_CLASS_ESCAPE \ special inside [...] + // w REG_NOSUB no subexpression match array + // x REG_LENIENT let some errors slide + // y REG_LEFT regexec() implicit ^... + // z REG_NULL NULL subexpressions ok + // $ expand C \c escapes in fields 2 and 3 + // / field 2 is a regsubcomp() expression + // = field 3 is a regdecomp() expression + // + // Field 1 control lines: + // + // C set LC_COLLATE and LC_CTYPE to locale in field 2 + // + // ?test ... output field 5 if passed and != EXPECTED, silent otherwise + // &test ... output field 5 if current and previous passed + // |test ... output field 5 if current passed and previous failed + // ; ... output field 2 if previous failed + // {test ... skip if failed until } + // } end of skip + // + // : comment comment copied as output NOTE + // :comment:test :comment: ignored + // N[OTE] comment comment copied as output NOTE + // T[EST] comment comment + // + // number use number for nmatch (20 by default) + flag := field[0] + switch flag[0] { + case '?', '&', '|', ';', '{', '}': + // Ignore all the control operators. + // Just run everything. + flag = flag[1:] + if flag == "" { + continue Reading + } + case ':': + i := strings.Index(flag[1:], ":") + if i < 0 { + t.Logf("skip: %s", line) + continue Reading + } + flag = flag[1+i+1:] + case 'C', 'N', 'T', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + t.Logf("skip: %s", line) + continue Reading + } + + // Can check field count now that we've handled the myriad comment formats. + if len(field) < 4 { + t.Errorf("%s:%d: too few fields: %s", file, lineno, line) + continue Reading + } + + // Expand C escapes (a.k.a. Go escapes). + if strings.Contains(flag, "$") { + f := `"` + field[1] + `"` + if field[1], err = strconv.Unquote(f); err != nil { + t.Errorf("%s:%d: cannot unquote %s", file, lineno, f) + } + f = `"` + field[2] + `"` + if field[2], err = strconv.Unquote(f); err != nil { + t.Errorf("%s:%d: cannot unquote %s", file, lineno, f) + } + } + + // Field 2: the regular expression pattern; SAME uses the pattern from + // the previous specification. + // + if field[1] == "SAME" { + field[1] = lastRegexp + } + lastRegexp = field[1] + + // Field 3: the string to match. + text := field[2] + + // Field 4: the test outcome... + ok, shouldCompile, shouldMatch, pos := parseFowlerResult(field[3]) + if !ok { + t.Errorf("%s:%d: cannot parse result %#q", file, lineno, field[3]) + continue Reading + } + + // Field 5: optional comment appended to the report. + + Testing: + // Run test once for each specified capital letter mode that we support. + for _, c := range flag { + pattern := field[1] + syn := syntax.POSIX | syntax.ClassNL + switch c { + default: + continue Testing + case 'E': + // extended regexp (what we support) + case 'L': + // literal + pattern = QuoteMeta(pattern) + } + + for _, c := range flag { + switch c { + case 'i': + syn |= syntax.FoldCase + } + } + + re, err := compile(pattern, syn, true) + if err != nil { + if shouldCompile { + t.Errorf("%s:%d: %#q did not compile", file, lineno, pattern) + } + continue Testing + } + if !shouldCompile { + t.Errorf("%s:%d: %#q should not compile", file, lineno, pattern) + continue Testing + } + match := re.MatchString(text) + if match != shouldMatch { + t.Errorf("%s:%d: %#q.Match(%#q) = %v, want %v", file, lineno, pattern, text, match, shouldMatch) + continue Testing + } + have := re.FindStringSubmatchIndex(text) + if (len(have) > 0) != match { + t.Errorf("%s:%d: %#q.Match(%#q) = %v, but %#q.FindSubmatchIndex(%#q) = %v", file, lineno, pattern, text, match, pattern, text, have) + continue Testing + } + if len(have) > len(pos) { + have = have[:len(pos)] + } + if !same(have, pos) { + t.Errorf("%s:%d: %#q.FindSubmatchIndex(%#q) = %v, want %v", file, lineno, pattern, text, have, pos) + } + } + } +} + +func parseFowlerResult(s string) (ok, compiled, matched bool, pos []int) { + // Field 4: the test outcome. This is either one of the posix error + // codes (with REG_ omitted) or the match array, a list of (m,n) + // entries with m and n being first and last+1 positions in the + // field 3 string, or NULL if REG_NOSUB is in effect and success + // is expected. BADPAT is acceptable in place of any regcomp(3) + // error code. The match[] array is initialized to (-2,-2) before + // each test. All array elements from 0 to nmatch-1 must be specified + // in the outcome. Unspecified endpoints (offset -1) are denoted by ?. + // Unset endpoints (offset -2) are denoted by X. {x}(o:n) denotes a + // matched (?{...}) expression, where x is the text enclosed by {...}, + // o is the expression ordinal counting from 1, and n is the length of + // the unmatched portion of the subject string. If x starts with a + // number then that is the return value of re_execf(), otherwise 0 is + // returned. + switch { + case s == "": + // Match with no position information. + ok = true + compiled = true + matched = true + return + case s == "NOMATCH": + // Match failure. + ok = true + compiled = true + matched = false + return + case 'A' <= s[0] && s[0] <= 'Z': + // All the other error codes are compile errors. + ok = true + compiled = false + return + } + compiled = true + + var x []int + for s != "" { + var end byte = ')' + if len(x)%2 == 0 { + if s[0] != '(' { + ok = false + return + } + s = s[1:] + end = ',' + } + i := 0 + for i < len(s) && s[i] != end { + i++ + } + if i == 0 || i == len(s) { + ok = false + return + } + var v = -1 + var err os.Error + if s[:i] != "?" { + v, err = strconv.Atoi(s[:i]) + if err != nil { + ok = false + return + } + } + x = append(x, v) + s = s[i+1:] + } + if len(x)%2 != 0 { + ok = false + return + } + ok = true + matched = true + pos = x + return +} + +var text []byte + +func makeText(n int) []byte { + if len(text) >= n { + return text[:n] + } + text = make([]byte, n) + for i := range text { + if rand.Intn(30) == 0 { + text[i] = '\n' + } else { + text[i] = byte(rand.Intn(0x7E+1-0x20) + 0x20) + } + } + return text +} + +func benchmark(b *testing.B, re string, n int) { + r := MustCompile(re) + t := makeText(n) + b.ResetTimer() + b.SetBytes(int64(n)) + for i := 0; i < b.N; i++ { + if r.Match(t) { + panic("match!") + } + } +} + +func benchold(b *testing.B, re string, n int) { + r := old.MustCompile(re) + t := makeText(n) + b.ResetTimer() + b.SetBytes(int64(n)) + for i := 0; i < b.N; i++ { + if r.Match(t) { + panic("match!") + } + } +} + +const ( + easy0 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ$" + easy1 = "A[AB]B[BC]C[CD]D[DE]E[EF]F[FG]G[GH]H[HI]I[IJ]J$" + medium = "[XYZ]ABCDEFGHIJKLMNOPQRSTUVWXYZ$" + hard = "[ -~]*ABCDEFGHIJKLMNOPQRSTUVWXYZ$" + parens = "([ -~])*(A)(B)(C)(D)(E)(F)(G)(H)(I)(J)(K)(L)(M)" + + "(N)(O)(P)(Q)(R)(S)(T)(U)(V)(W)(X)(Y)(Z)$" +) + +func BenchmarkMatchEasy0_1K(b *testing.B) { benchmark(b, easy0, 1<<10) } +func BenchmarkMatchEasy0_1K_Old(b *testing.B) { benchold(b, easy0, 1<<10) } +func BenchmarkMatchEasy0_1M(b *testing.B) { benchmark(b, easy0, 1<<20) } +func BenchmarkMatchEasy0_1M_Old(b *testing.B) { benchold(b, easy0, 1<<20) } +func BenchmarkMatchEasy0_32K(b *testing.B) { benchmark(b, easy0, 32<<10) } +func BenchmarkMatchEasy0_32K_Old(b *testing.B) { benchold(b, easy0, 32<<10) } +func BenchmarkMatchEasy0_32M(b *testing.B) { benchmark(b, easy0, 32<<20) } +func BenchmarkMatchEasy0_32M_Old(b *testing.B) { benchold(b, easy0, 32<<20) } +func BenchmarkMatchEasy1_1K(b *testing.B) { benchmark(b, easy1, 1<<10) } +func BenchmarkMatchEasy1_1K_Old(b *testing.B) { benchold(b, easy1, 1<<10) } +func BenchmarkMatchEasy1_1M(b *testing.B) { benchmark(b, easy1, 1<<20) } +func BenchmarkMatchEasy1_1M_Old(b *testing.B) { benchold(b, easy1, 1<<20) } +func BenchmarkMatchEasy1_32K(b *testing.B) { benchmark(b, easy1, 32<<10) } +func BenchmarkMatchEasy1_32K_Old(b *testing.B) { benchold(b, easy1, 32<<10) } +func BenchmarkMatchEasy1_32M(b *testing.B) { benchmark(b, easy1, 32<<20) } +func BenchmarkMatchEasy1_32M_Old(b *testing.B) { benchold(b, easy1, 32<<20) } +func BenchmarkMatchMedium_1K(b *testing.B) { benchmark(b, medium, 1<<10) } +func BenchmarkMatchMedium_1K_Old(b *testing.B) { benchold(b, medium, 1<<10) } +func BenchmarkMatchMedium_1M(b *testing.B) { benchmark(b, medium, 1<<20) } +func BenchmarkMatchMedium_1M_Old(b *testing.B) { benchold(b, medium, 1<<20) } +func BenchmarkMatchMedium_32K(b *testing.B) { benchmark(b, medium, 32<<10) } +func BenchmarkMatchMedium_32K_Old(b *testing.B) { benchold(b, medium, 32<<10) } +func BenchmarkMatchMedium_32M(b *testing.B) { benchmark(b, medium, 32<<20) } +func BenchmarkMatchMedium_32M_Old(b *testing.B) { benchold(b, medium, 32<<20) } +func BenchmarkMatchHard_1K(b *testing.B) { benchmark(b, hard, 1<<10) } +func BenchmarkMatchHard_1K_Old(b *testing.B) { benchold(b, hard, 1<<10) } +func BenchmarkMatchHard_1M(b *testing.B) { benchmark(b, hard, 1<<20) } +func BenchmarkMatchHard_1M_Old(b *testing.B) { benchold(b, hard, 1<<20) } +func BenchmarkMatchHard_32K(b *testing.B) { benchmark(b, hard, 32<<10) } +func BenchmarkMatchHard_32K_Old(b *testing.B) { benchold(b, hard, 32<<10) } +func BenchmarkMatchHard_32M(b *testing.B) { benchmark(b, hard, 32<<20) } +func BenchmarkMatchHard_32M_Old(b *testing.B) { benchold(b, hard, 32<<20) } diff --git a/libgo/go/regexp/find_test.go b/libgo/go/regexp/find_test.go index 83b249e..e07eb7d 100644 --- a/libgo/go/regexp/find_test.go +++ b/libgo/go/regexp/find_test.go @@ -58,8 +58,8 @@ var findTests = []FindTest{ {`(([^xyz]*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 3, 4)}, {`((a|b|c)*(d))`, "abcd", build(1, 0, 4, 0, 4, 2, 3, 3, 4)}, {`(((a|b|c)*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 2, 3, 3, 4)}, - {`\a\b\f\n\r\t\v`, "\a\b\f\n\r\t\v", build(1, 0, 7)}, - {`[\a\b\f\n\r\t\v]+`, "\a\b\f\n\r\t\v", build(1, 0, 7)}, + {`\a\f\n\r\t\v`, "\a\f\n\r\t\v", build(1, 0, 6)}, + {`[\a\f\n\r\t\v]+`, "\a\f\n\r\t\v", build(1, 0, 6)}, {`a*(|(b))c*`, "aacc", build(1, 0, 4, 2, 2, -1, -1)}, {`(.*).*`, "ab", build(1, 0, 2, 0, 2)}, @@ -80,6 +80,32 @@ var findTests = []FindTest{ {`data`, "daXY data", build(1, 5, 9)}, {`da(.)a$`, "daXY data", build(1, 5, 9, 7, 8)}, {`zx+`, "zzx", build(1, 1, 3)}, + {`ab$`, "abcab", build(1, 3, 5)}, + {`(aa)*$`, "a", build(1, 1, 1, -1, -1)}, + {`(?:.|(?:.a))`, "", nil}, + {`(?:A(?:A|a))`, "Aa", build(1, 0, 2)}, + {`(?:A|(?:A|a))`, "a", build(1, 0, 1)}, + {`(a){0}`, "", build(1, 0, 0, -1, -1)}, + {`(?-s)(?:(?:^).)`, "\n", nil}, + {`(?s)(?:(?:^).)`, "\n", build(1, 0, 1)}, + {`(?:(?:^).)`, "\n", nil}, + {`\b`, "x", build(2, 0, 0, 1, 1)}, + {`\b`, "xx", build(2, 0, 0, 2, 2)}, + {`\b`, "x y", build(4, 0, 0, 1, 1, 2, 2, 3, 3)}, + {`\b`, "xx yy", build(4, 0, 0, 2, 2, 3, 3, 5, 5)}, + {`\B`, "x", nil}, + {`\B`, "xx", build(1, 1, 1)}, + {`\B`, "x y", nil}, + {`\B`, "xx yy", build(2, 1, 1, 4, 4)}, + + // RE2 tests + {`[^\S\s]`, "abcd", nil}, + {`[^\S[:space:]]`, "abcd", nil}, + {`[^\D\d]`, "abcd", nil}, + {`[^\D[:digit:]]`, "abcd", nil}, + {`(?i)\W`, "x", nil}, + {`(?i)\W`, "k", nil}, + {`(?i)\W`, "s", nil}, // can backslash-escape any punctuation {`\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\{\|\}\~`, @@ -209,7 +235,7 @@ func TestFindAll(t *testing.T) { case test.matches == nil && result != nil: t.Errorf("expected no match; got one: %s", test) case test.matches != nil && result == nil: - t.Errorf("expected match; got none: %s", test) + t.Fatalf("expected match; got none: %s", test) case test.matches != nil && result != nil: if len(test.matches) != len(result) { t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) diff --git a/libgo/go/regexp/regexp.go b/libgo/go/regexp/regexp.go index e8d4c08..2325f62 100644 --- a/libgo/go/regexp/regexp.go +++ b/libgo/go/regexp/regexp.go @@ -3,27 +3,12 @@ // Package regexp implements a simple regular expression library. // -// The syntax of the regular expressions accepted is: +// The syntax of the regular expressions accepted is the same +// general syntax used by Perl, Python, and other languages. +// More precisely, it is the syntax accepted by RE2 and described at +// http://code.google.com/p/re2/wiki/Syntax, except for \C. // -// regexp: -// concatenation { '|' concatenation } -// concatenation: -// { closure } -// closure: -// term [ '*' | '+' | '?' ] -// term: -// '^' -// '$' -// '.' -// character -// '[' [ '^' ] { character-range } ']' -// '(' regexp ')' -// character-range: -// character [ '-' character ] -// -// All characters are UTF-8-encoded code points. Backslashes escape special -// characters, including inside character classes. The standard Go character -// escapes are also recognized: \a \b \f \n \r \t \v. +// All characters are UTF-8-encoded code points. // // There are 16 methods of Regexp that match a regular expression and identify // the matched text. Their names are matched by this regular expression: @@ -72,7 +57,10 @@ import ( "bytes" "io" "os" + "regexp/syntax" + "strconv" "strings" + "sync" "utf8" ) @@ -85,528 +73,24 @@ func (e Error) String() string { return string(e) } -// Error codes returned by failures to parse an expression. -var ( - ErrInternal = Error("regexp: internal error") - ErrUnmatchedLpar = Error("regexp: unmatched '('") - ErrUnmatchedRpar = Error("regexp: unmatched ')'") - ErrUnmatchedLbkt = Error("regexp: unmatched '['") - ErrUnmatchedRbkt = Error("regexp: unmatched ']'") - ErrBadRange = Error("regexp: bad range in character class") - ErrExtraneousBackslash = Error("regexp: extraneous backslash") - ErrBadClosure = Error("regexp: repeated closure (**, ++, etc.)") - ErrBareClosure = Error("regexp: closure applies to nothing") - ErrBadBackslash = Error("regexp: illegal backslash escape") -) - -const ( - iStart = iota // beginning of program - iEnd // end of program: success - iBOT // '^' beginning of text - iEOT // '$' end of text - iChar // 'a' regular character - iCharClass // [a-z] character class - iAny // '.' any character including newline - iNotNL // [^\n] special case: any character but newline - iBra // '(' parenthesized expression: 2*braNum for left, 2*braNum+1 for right - iAlt // '|' alternation - iNop // do nothing; makes it easy to link without patching -) - -// An instruction executed by the NFA -type instr struct { - kind int // the type of this instruction: iChar, iAny, etc. - index int // used only in debugging; could be eliminated - next *instr // the instruction to execute after this one - // Special fields valid only for some items. - char int // iChar - braNum int // iBra, iEbra - cclass *charClass // iCharClass - left *instr // iAlt, other branch -} - -func (i *instr) print() { - switch i.kind { - case iStart: - print("start") - case iEnd: - print("end") - case iBOT: - print("bot") - case iEOT: - print("eot") - case iChar: - print("char ", string(i.char)) - case iCharClass: - i.cclass.print() - case iAny: - print("any") - case iNotNL: - print("notnl") - case iBra: - if i.braNum&1 == 0 { - print("bra", i.braNum/2) - } else { - print("ebra", i.braNum/2) - } - case iAlt: - print("alt(", i.left.index, ")") - case iNop: - print("nop") - } -} - // Regexp is the representation of a compiled regular expression. // The public interface is entirely through methods. // A Regexp is safe for concurrent use by multiple goroutines. type Regexp struct { - expr string // the original expression - prefix string // initial plain text string - prefixBytes []byte // initial plain text bytes - inst []*instr - start *instr // first instruction of machine - prefixStart *instr // where to start if there is a prefix - nbra int // number of brackets in expression, for subexpressions -} - -type charClass struct { - negate bool // is character class negated? ([^a-z]) - // slice of int, stored pairwise: [a-z] is (a,z); x is (x,x): - ranges []int - cmin, cmax int -} - -func (cclass *charClass) print() { - print("charclass") - if cclass.negate { - print(" (negated)") - } - for i := 0; i < len(cclass.ranges); i += 2 { - l := cclass.ranges[i] - r := cclass.ranges[i+1] - if l == r { - print(" [", string(l), "]") - } else { - print(" [", string(l), "-", string(r), "]") - } - } -} - -func (cclass *charClass) addRange(a, b int) { - // range is a through b inclusive - cclass.ranges = append(cclass.ranges, a, b) - if a < cclass.cmin { - cclass.cmin = a - } - if b > cclass.cmax { - cclass.cmax = b - } -} - -func (cclass *charClass) matches(c int) bool { - if c < cclass.cmin || c > cclass.cmax { - return cclass.negate - } - ranges := cclass.ranges - for i := 0; i < len(ranges); i = i + 2 { - if ranges[i] <= c && c <= ranges[i+1] { - return !cclass.negate - } - } - return cclass.negate -} - -func newCharClass() *instr { - i := &instr{kind: iCharClass} - i.cclass = new(charClass) - i.cclass.ranges = make([]int, 0, 4) - i.cclass.cmin = 0x10FFFF + 1 // MaxRune + 1 - i.cclass.cmax = -1 - return i -} - -func (re *Regexp) add(i *instr) *instr { - i.index = len(re.inst) - re.inst = append(re.inst, i) - return i -} - -type parser struct { - re *Regexp - nlpar int // number of unclosed lpars - pos int - ch int -} - -func (p *parser) error(err Error) { - panic(err) -} - -const endOfText = -1 - -func (p *parser) c() int { return p.ch } - -func (p *parser) nextc() int { - if p.pos >= len(p.re.expr) { - p.ch = endOfText - } else { - c, w := utf8.DecodeRuneInString(p.re.expr[p.pos:]) - p.ch = c - p.pos += w - } - return p.ch -} - -func newParser(re *Regexp) *parser { - p := new(parser) - p.re = re - p.nextc() // load p.ch - return p -} - -func special(c int) bool { - for _, r := range `\.+*?()|[]^$` { - if c == r { - return true - } - } - return false -} - -func ispunct(c int) bool { - for _, r := range "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" { - if c == r { - return true - } - } - return false -} - -var escapes = []byte("abfnrtv") -var escaped = []byte("\a\b\f\n\r\t\v") - -func escape(c int) int { - for i, b := range escapes { - if int(b) == c { - return i - } - } - return -1 -} - -func (p *parser) checkBackslash() int { - c := p.c() - if c == '\\' { - c = p.nextc() - switch { - case c == endOfText: - p.error(ErrExtraneousBackslash) - case ispunct(c): - // c is as delivered - case escape(c) >= 0: - c = int(escaped[escape(c)]) - default: - p.error(ErrBadBackslash) - } - } - return c -} - -func (p *parser) charClass() *instr { - i := newCharClass() - cc := i.cclass - if p.c() == '^' { - cc.negate = true - p.nextc() - } - left := -1 - for { - switch c := p.c(); c { - case ']', endOfText: - if left >= 0 { - p.error(ErrBadRange) - } - // Is it [^\n]? - if cc.negate && len(cc.ranges) == 2 && - cc.ranges[0] == '\n' && cc.ranges[1] == '\n' { - nl := &instr{kind: iNotNL} - p.re.add(nl) - return nl - } - // Special common case: "[a]" -> "a" - if !cc.negate && len(cc.ranges) == 2 && cc.ranges[0] == cc.ranges[1] { - c := &instr{kind: iChar, char: cc.ranges[0]} - p.re.add(c) - return c - } - p.re.add(i) - return i - case '-': // do this before backslash processing - p.error(ErrBadRange) - default: - c = p.checkBackslash() - p.nextc() - switch { - case left < 0: // first of pair - if p.c() == '-' { // range - p.nextc() - left = c - } else { // single char - cc.addRange(c, c) - } - case left <= c: // second of pair - cc.addRange(left, c) - left = -1 - default: - p.error(ErrBadRange) - } - } - } - panic("unreachable") -} - -func (p *parser) term() (start, end *instr) { - switch c := p.c(); c { - case '|', endOfText: - return nil, nil - case '*', '+', '?': - p.error(ErrBareClosure) - case ')': - if p.nlpar == 0 { - p.error(ErrUnmatchedRpar) - } - return nil, nil - case ']': - p.error(ErrUnmatchedRbkt) - case '^': - p.nextc() - start = p.re.add(&instr{kind: iBOT}) - return start, start - case '$': - p.nextc() - start = p.re.add(&instr{kind: iEOT}) - return start, start - case '.': - p.nextc() - start = p.re.add(&instr{kind: iAny}) - return start, start - case '[': - p.nextc() - start = p.charClass() - if p.c() != ']' { - p.error(ErrUnmatchedLbkt) - } - p.nextc() - return start, start - case '(': - p.nextc() - p.nlpar++ - p.re.nbra++ // increment first so first subexpr is \1 - nbra := p.re.nbra - start, end = p.regexp() - if p.c() != ')' { - p.error(ErrUnmatchedLpar) - } - p.nlpar-- - p.nextc() - bra := &instr{kind: iBra, braNum: 2 * nbra} - p.re.add(bra) - ebra := &instr{kind: iBra, braNum: 2*nbra + 1} - p.re.add(ebra) - if start == nil { - if end == nil { - p.error(ErrInternal) - return - } - start = ebra - } else { - end.next = ebra - } - bra.next = start - return bra, ebra - default: - c = p.checkBackslash() - p.nextc() - start = &instr{kind: iChar, char: c} - p.re.add(start) - return start, start - } - panic("unreachable") -} - -func (p *parser) closure() (start, end *instr) { - start, end = p.term() - if start == nil { - return - } - switch p.c() { - case '*': - // (start,end)*: - alt := &instr{kind: iAlt} - p.re.add(alt) - end.next = alt // after end, do alt - alt.left = start // alternate brach: return to start - start = alt // alt becomes new (start, end) - end = alt - case '+': - // (start,end)+: - alt := &instr{kind: iAlt} - p.re.add(alt) - end.next = alt // after end, do alt - alt.left = start // alternate brach: return to start - end = alt // start is unchanged; end is alt - case '?': - // (start,end)?: - alt := &instr{kind: iAlt} - p.re.add(alt) - nop := &instr{kind: iNop} - p.re.add(nop) - alt.left = start // alternate branch is start - alt.next = nop // follow on to nop - end.next = nop // after end, go to nop - start = alt // start is now alt - end = nop // end is nop pointed to by both branches - default: - return - } - switch p.nextc() { - case '*', '+', '?': - p.error(ErrBadClosure) - } - return -} - -func (p *parser) concatenation() (start, end *instr) { - for { - nstart, nend := p.closure() - switch { - case nstart == nil: // end of this concatenation - if start == nil { // this is the empty string - nop := p.re.add(&instr{kind: iNop}) - return nop, nop - } - return - case start == nil: // this is first element of concatenation - start, end = nstart, nend - default: - end.next = nstart - end = nend - } - } - panic("unreachable") -} - -func (p *parser) regexp() (start, end *instr) { - start, end = p.concatenation() - for { - switch p.c() { - default: - return - case '|': - p.nextc() - nstart, nend := p.concatenation() - alt := &instr{kind: iAlt} - p.re.add(alt) - alt.left = start - alt.next = nstart - nop := &instr{kind: iNop} - p.re.add(nop) - end.next = nop - nend.next = nop - start, end = alt, nop - } - } - panic("unreachable") -} - -func unNop(i *instr) *instr { - for i.kind == iNop { - i = i.next - } - return i -} - -func (re *Regexp) eliminateNops() { - for _, inst := range re.inst { - if inst.kind == iEnd { - continue - } - inst.next = unNop(inst.next) - if inst.kind == iAlt { - inst.left = unNop(inst.left) - } - } -} - -func (re *Regexp) dump() { - print("prefix <", re.prefix, ">\n") - for _, inst := range re.inst { - print(inst.index, ": ") - inst.print() - if inst.kind != iEnd { - print(" -> ", inst.next.index) - } - print("\n") - } -} - -func (re *Regexp) doParse() { - p := newParser(re) - start := &instr{kind: iStart} - re.add(start) - s, e := p.regexp() - start.next = s - re.start = start - e.next = re.add(&instr{kind: iEnd}) - - if debug { - re.dump() - println() - } - - re.eliminateNops() - if debug { - re.dump() - println() - } - re.setPrefix() - if debug { - re.dump() - println() - } -} - -// Extract regular text from the beginning of the pattern, -// possibly after a leading iBOT. -// That text can be used by doExecute to speed up matching. -func (re *Regexp) setPrefix() { - var b []byte - var utf = make([]byte, utf8.UTFMax) - var inst *instr - // First instruction is start; skip that. Also skip any initial iBOT. - inst = re.inst[0].next - for inst.kind == iBOT { - inst = inst.next - } -Loop: - for ; inst.kind != iEnd; inst = inst.next { - // stop if this is not a char - if inst.kind != iChar { - break - } - // stop if this char can be followed by a match for an empty string, - // which includes closures, ^, and $. - switch inst.next.kind { - case iBOT, iEOT, iAlt: - break Loop - } - n := utf8.EncodeRune(utf, inst.char) - b = append(b, utf[0:n]...) - } - // point prefixStart instruction to first non-CHAR after prefix - re.prefixStart = inst - re.prefixBytes = b - re.prefix = string(b) + // read-only after Compile + expr string // as passed to Compile + prog *syntax.Prog // compiled program + prefix string // required prefix in unanchored matches + prefixBytes []byte // prefix, as a []byte + prefixComplete bool // prefix is the entire regexp + prefixRune int // first rune in prefix + cond syntax.EmptyOp // empty-width conditions required at start of match + numSubexp int + longest bool + + // cache of machines for running regexp + mu sync.Mutex + machine []*machine } // String returns the source text used to compile the regular expression. @@ -614,21 +98,96 @@ func (re *Regexp) String() string { return re.expr } -// Compile parses a regular expression and returns, if successful, a Regexp -// object that can be used to match against text. -func Compile(str string) (regexp *Regexp, error os.Error) { - regexp = new(Regexp) - // doParse will panic if there is a parse error. - defer func() { - if e := recover(); e != nil { - regexp = nil - error = e.(Error) // Will re-panic if error was not an Error, e.g. nil-pointer exception - } - }() - regexp.expr = str - regexp.inst = make([]*instr, 0, 10) - regexp.doParse() - return +// Compile parses a regular expression and returns, if successful, +// a Regexp object that can be used to match against text. +// +// When matching against text, the regexp returns a match that +// begins as early as possible in the input (leftmost), and among those +// it chooses the one that a backtracking search would have found first. +// This so-called leftmost-first matching is the same semantics +// that Perl, Python, and other implementations use, although this +// package implements it without the expense of backtracking. +// For POSIX leftmost-longest matching, see CompilePOSIX. +func Compile(expr string) (*Regexp, os.Error) { + return compile(expr, syntax.Perl, false) +} + +// CompilePOSIX is like Compile but restricts the regular expression +// to POSIX ERE (egrep) syntax and changes the match semantics to +// leftmost-longest. +// +// That is, when matching against text, the regexp returns a match that +// begins as early as possible in the input (leftmost), and among those +// it chooses a match that is as long as possible. +// This so-called leftmost-longest matching is the same semantics +// that early regular expression implementations used and that POSIX +// specifies. +// +// However, there can be multiple leftmost-longest matches, with different +// submatch choices, and here this package diverges from POSIX. +// Among the possible leftmost-longest matches, this package chooses +// the one that a backtracking search would have found first, while POSIX +// specifies that the match be chosen to maximize the length of the first +// subexpression, then the second, and so on from left to right. +// The POSIX rule is computationally prohibitive and not even well-defined. +// See http://swtch.com/~rsc/regexp/regexp2.html#posix for details. +func CompilePOSIX(expr string) (*Regexp, os.Error) { + return compile(expr, syntax.POSIX, true) +} + +func compile(expr string, mode syntax.Flags, longest bool) (*Regexp, os.Error) { + re, err := syntax.Parse(expr, mode) + if err != nil { + return nil, err + } + maxCap := re.MaxCap() + re = re.Simplify() + prog, err := syntax.Compile(re) + if err != nil { + return nil, err + } + regexp := &Regexp{ + expr: expr, + prog: prog, + numSubexp: maxCap, + cond: prog.StartCond(), + longest: longest, + } + regexp.prefix, regexp.prefixComplete = prog.Prefix() + if regexp.prefix != "" { + // TODO(rsc): Remove this allocation by adding + // IndexString to package bytes. + regexp.prefixBytes = []byte(regexp.prefix) + regexp.prefixRune, _ = utf8.DecodeRuneInString(regexp.prefix) + } + return regexp, nil +} + +// get returns a machine to use for matching re. +// It uses the re's machine cache if possible, to avoid +// unnecessary allocation. +func (re *Regexp) get() *machine { + re.mu.Lock() + if n := len(re.machine); n > 0 { + z := re.machine[n-1] + re.machine = re.machine[:n-1] + re.mu.Unlock() + return z + } + re.mu.Unlock() + z := progMachine(re.prog) + z.re = re + return z +} + +// put returns a machine to the re's machine cache. +// There is no attempt to limit the size of the cache, so it will +// grow to the maximum number of simultaneous matches +// run using re. (The cache empties when re gets garbage collected.) +func (re *Regexp) put(z *machine) { + re.mu.Lock() + re.machine = append(re.machine, z) + re.mu.Unlock() } // MustCompile is like Compile but panics if the expression cannot be parsed. @@ -637,116 +196,35 @@ func Compile(str string) (regexp *Regexp, error os.Error) { func MustCompile(str string) *Regexp { regexp, error := Compile(str) if error != nil { - panic(`regexp: compiling "` + str + `": ` + error.String()) + panic(`regexp: Compile(` + quote(str) + `): ` + error.String()) } return regexp } -// NumSubexp returns the number of parenthesized subexpressions in this Regexp. -func (re *Regexp) NumSubexp() int { return re.nbra } - -// The match arena allows us to reduce the garbage generated by tossing -// match vectors away as we execute. Matches are ref counted and returned -// to a free list when no longer active. Increases a simple benchmark by 22X. -type matchArena struct { - head *matchVec - len int // length of match vector - pos int - atBOT bool // whether we're at beginning of text - atEOT bool // whether we're at end of text -} - -type matchVec struct { - m []int // pairs of bracketing submatches. 0th is start,end - ref int - next *matchVec -} - -func (a *matchArena) new() *matchVec { - if a.head == nil { - const N = 10 - block := make([]matchVec, N) - for i := 0; i < N; i++ { - b := &block[i] - b.next = a.head - a.head = b - } - } - m := a.head - a.head = m.next - m.ref = 0 - if m.m == nil { - m.m = make([]int, a.len) - } - return m -} - -func (a *matchArena) free(m *matchVec) { - m.ref-- - if m.ref == 0 { - m.next = a.head - a.head = m +// MustCompilePOSIX is like CompilePOSIX but panics if the expression cannot be parsed. +// It simplifies safe initialization of global variables holding compiled regular +// expressions. +func MustCompilePOSIX(str string) *Regexp { + regexp, error := CompilePOSIX(str) + if error != nil { + panic(`regexp: CompilePOSIX(` + quote(str) + `): ` + error.String()) } + return regexp } -func (a *matchArena) copy(m *matchVec) *matchVec { - m1 := a.new() - copy(m1.m, m.m) - return m1 -} - -func (a *matchArena) noMatch() *matchVec { - m := a.new() - for i := range m.m { - m.m[i] = -1 // no match seen; catches cases like "a(b)?c" on "ac" +func quote(s string) string { + if strconv.CanBackquote(s) { + return "`" + s + "`" } - m.ref = 1 - return m + return strconv.Quote(s) } -type state struct { - inst *instr // next instruction to execute - prefixed bool // this match began with a fixed prefix - match *matchVec +// NumSubexp returns the number of parenthesized subexpressions in this Regexp. +func (re *Regexp) NumSubexp() int { + return re.numSubexp } -// Append new state to to-do list. Leftmost-longest wins so avoid -// adding a state that's already active. The matchVec will be inc-ref'ed -// if it is assigned to a state. -func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matchVec) []state { - switch inst.kind { - case iBOT: - if a.atBOT { - s = a.addState(s, inst.next, prefixed, match) - } - return s - case iEOT: - if a.atEOT { - s = a.addState(s, inst.next, prefixed, match) - } - return s - case iBra: - match.m[inst.braNum] = a.pos - s = a.addState(s, inst.next, prefixed, match) - return s - } - l := len(s) - // States are inserted in order so it's sufficient to see if we have the same - // instruction; no need to see if existing match is earlier (it is). - for i := 0; i < l; i++ { - if s[i].inst == inst { - return s - } - } - s = append(s, state{inst, prefixed, match}) - match.ref++ - if inst.kind == iAlt { - s = a.addState(s, inst.left, prefixed, a.copy(match)) - // give other branch a copy of this match vector - s = a.addState(s, inst.next, prefixed, a.copy(match)) - } - return s -} +const endOfText = -1 // input abstracts different representations of the input text. It provides // one-character lookahead. @@ -755,6 +233,7 @@ type input interface { canCheckPrefix() bool // can we look ahead without losing info? hasPrefix(re *Regexp) bool index(re *Regexp, pos int) int + context(pos int) syntax.EmptyOp } // inputString scans a string. @@ -768,7 +247,11 @@ func newInputString(str string) *inputString { func (i *inputString) step(pos int) (int, int) { if pos < len(i.str) { - return utf8.DecodeRuneInString(i.str[pos:len(i.str)]) + c := i.str[pos] + if c < utf8.RuneSelf { + return int(c), 1 + } + return utf8.DecodeRuneInString(i.str[pos:]) } return endOfText, 0 } @@ -785,6 +268,17 @@ func (i *inputString) index(re *Regexp, pos int) int { return strings.Index(i.str[pos:], re.prefix) } +func (i *inputString) context(pos int) syntax.EmptyOp { + r1, r2 := -1, -1 + if pos > 0 && pos <= len(i.str) { + r1, _ = utf8.DecodeLastRuneInString(i.str[:pos]) + } + if pos < len(i.str) { + r2, _ = utf8.DecodeRuneInString(i.str[pos:]) + } + return syntax.EmptyOpContext(r1, r2) +} + // inputBytes scans a byte slice. type inputBytes struct { str []byte @@ -796,7 +290,11 @@ func newInputBytes(str []byte) *inputBytes { func (i *inputBytes) step(pos int) (int, int) { if pos < len(i.str) { - return utf8.DecodeRune(i.str[pos:len(i.str)]) + c := i.str[pos] + if c < utf8.RuneSelf { + return int(c), 1 + } + return utf8.DecodeRune(i.str[pos:]) } return endOfText, 0 } @@ -813,6 +311,17 @@ func (i *inputBytes) index(re *Regexp, pos int) int { return bytes.Index(i.str[pos:], re.prefixBytes) } +func (i *inputBytes) context(pos int) syntax.EmptyOp { + r1, r2 := -1, -1 + if pos > 0 && pos <= len(i.str) { + r1, _ = utf8.DecodeLastRune(i.str[:pos]) + } + if pos < len(i.str) { + r2, _ = utf8.DecodeRune(i.str[pos:]) + } + return syntax.EmptyOpContext(r1, r2) +} + // inputReader scans a RuneReader. type inputReader struct { r io.RuneReader @@ -850,150 +359,35 @@ func (i *inputReader) index(re *Regexp, pos int) int { return -1 } -// Search match starting from pos bytes into the input. -func (re *Regexp) doExecute(i input, pos int) []int { - var s [2][]state - s[0] = make([]state, 0, 10) - s[1] = make([]state, 0, 10) - in, out := 0, 1 - var final state - found := false - anchored := re.inst[0].next.kind == iBOT - if anchored && pos > 0 { - return nil - } - // fast check for initial plain substring - if i.canCheckPrefix() && re.prefix != "" { - advance := 0 - if anchored { - if !i.hasPrefix(re) { - return nil - } - } else { - advance = i.index(re, pos) - if advance == -1 { - return nil - } - } - pos += advance - } - // We look one character ahead so we can match $, which checks whether - // we are at EOT. - nextChar, nextWidth := i.step(pos) - arena := &matchArena{ - len: 2 * (re.nbra + 1), - pos: pos, - atBOT: pos == 0, - atEOT: nextChar == endOfText, - } - for c, startPos := 0, pos; c != endOfText; { - if !found && (pos == startPos || !anchored) { - // prime the pump if we haven't seen a match yet - match := arena.noMatch() - match.m[0] = pos - s[out] = arena.addState(s[out], re.start.next, false, match) - arena.free(match) // if addState saved it, ref was incremented - } else if len(s[out]) == 0 { - // machine has completed - break - } - in, out = out, in // old out state is new in state - // clear out old state - old := s[out] - for _, state := range old { - arena.free(state.match) - } - s[out] = old[0:0] // truncate state vector - c = nextChar - thisPos := pos - pos += nextWidth - nextChar, nextWidth = i.step(pos) - arena.atEOT = nextChar == endOfText - arena.atBOT = false - arena.pos = pos - for _, st := range s[in] { - switch st.inst.kind { - case iBOT: - case iEOT: - case iChar: - if c == st.inst.char { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) - } - case iCharClass: - if st.inst.cclass.matches(c) { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) - } - case iAny: - if c != endOfText { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) - } - case iNotNL: - if c != endOfText && c != '\n' { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) - } - case iBra: - case iAlt: - case iEnd: - // choose leftmost longest - if !found || // first - st.match.m[0] < final.match.m[0] || // leftmost - (st.match.m[0] == final.match.m[0] && thisPos > final.match.m[1]) { // longest - if final.match != nil { - arena.free(final.match) - } - final = st - final.match.ref++ - final.match.m[1] = thisPos - } - found = true - default: - st.inst.print() - panic("unknown instruction in execute") - } - } - } - if final.match == nil { - return nil - } - // if match found, back up start of match by width of prefix. - if final.prefixed && len(final.match.m) > 0 { - final.match.m[0] -= len(re.prefix) - } - return final.match.m +func (i *inputReader) context(pos int) syntax.EmptyOp { + return 0 } // LiteralPrefix returns a literal string that must begin any match // of the regular expression re. It returns the boolean true if the // literal string comprises the entire regular expression. func (re *Regexp) LiteralPrefix() (prefix string, complete bool) { - c := make([]int, len(re.inst)-2) // minus start and end. - // First instruction is start; skip that. - i := 0 - for inst := re.inst[0].next; inst.kind != iEnd; inst = inst.next { - // stop if this is not a char - if inst.kind != iChar { - return string(c[:i]), false - } - c[i] = inst.char - i++ - } - return string(c[:i]), true + return re.prefix, re.prefixComplete } // MatchReader returns whether the Regexp matches the text read by the // RuneReader. The return value is a boolean: true for match, false for no // match. func (re *Regexp) MatchReader(r io.RuneReader) bool { - return len(re.doExecute(newInputReader(r), 0)) > 0 + return re.doExecute(newInputReader(r), 0, 0) != nil } // MatchString returns whether the Regexp matches the string s. // The return value is a boolean: true for match, false for no match. -func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(newInputString(s), 0)) > 0 } +func (re *Regexp) MatchString(s string) bool { + return re.doExecute(newInputString(s), 0, 0) != nil +} // Match returns whether the Regexp matches the byte slice b. // The return value is a boolean: true for match, false for no match. -func (re *Regexp) Match(b []byte) bool { return len(re.doExecute(newInputBytes(b), 0)) > 0 } +func (re *Regexp) Match(b []byte) bool { + return re.doExecute(newInputBytes(b), 0, 0) != nil +} // MatchReader checks whether a textual regular expression matches the text // read by the RuneReader. More complicated queries need to use Compile and @@ -1044,7 +438,7 @@ func (re *Regexp) ReplaceAllStringFunc(src string, repl func(string) string) str searchPos := 0 // position where we next look for a match buf := new(bytes.Buffer) for searchPos <= len(src) { - a := re.doExecute(newInputString(src), searchPos) + a := re.doExecute(newInputString(src), searchPos, 2) if len(a) == 0 { break // no more matches } @@ -1096,7 +490,7 @@ func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { searchPos := 0 // position where we next look for a match buf := new(bytes.Buffer) for searchPos <= len(src) { - a := re.doExecute(newInputBytes(src), searchPos) + a := re.doExecute(newInputBytes(src), searchPos, 2) if len(a) == 0 { break // no more matches } @@ -1132,6 +526,12 @@ func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { return buf.Bytes() } +var specialBytes = []byte(`\.+*?()|[]{}^$`) + +func special(b byte) bool { + return bytes.IndexByte(specialBytes, b) >= 0 +} + // QuoteMeta returns a string that quotes all regular expression metacharacters // inside the argument text; the returned string is a regular expression matching // the literal text. For example, QuoteMeta(`[foo]`) returns `\[foo\]`. @@ -1141,7 +541,7 @@ func QuoteMeta(s string) string { // A byte loop is correct because all metacharacters are ASCII. j := 0 for i := 0; i < len(s); i++ { - if special(int(s[i])) { + if special(s[i]) { b[j] = '\\' j++ } @@ -1151,6 +551,23 @@ func QuoteMeta(s string) string { return string(b[0:j]) } +// The number of capture values in the program may correspond +// to fewer capturing expressions than are in the regexp. +// For example, "(a){0}" turns into an empty program, so the +// maximum capture in the program is 0 but we need to return +// an expression for \1. Pad appends -1s to the slice a as needed. +func (re *Regexp) pad(a []int) []int { + if a == nil { + // No match. + return nil + } + n := (1 + re.numSubexp) * 2 + for len(a) < n { + a = append(a, -1) + } + return a +} + // Find matches in slice b if b is non-nil, otherwise find matches in string s. func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { var end int @@ -1167,7 +584,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { } else { in = newInputBytes(b) } - matches := re.doExecute(in, pos) + matches := re.doExecute(in, pos, re.prog.NumCap) if len(matches) == 0 { break } @@ -1198,7 +615,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { prevMatchEnd = matches[1] if accept { - deliver(matches) + deliver(re.pad(matches)) i++ } } @@ -1207,7 +624,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { // Find returns a slice holding the text of the leftmost match in b of the regular expression. // A return value of nil indicates no match. func (re *Regexp) Find(b []byte) []byte { - a := re.doExecute(newInputBytes(b), 0) + a := re.doExecute(newInputBytes(b), 0, 2) if a == nil { return nil } @@ -1219,7 +636,7 @@ func (re *Regexp) Find(b []byte) []byte { // b[loc[0]:loc[1]]. // A return value of nil indicates no match. func (re *Regexp) FindIndex(b []byte) (loc []int) { - a := re.doExecute(newInputBytes(b), 0) + a := re.doExecute(newInputBytes(b), 0, 2) if a == nil { return nil } @@ -1232,7 +649,7 @@ func (re *Regexp) FindIndex(b []byte) (loc []int) { // an empty string. Use FindStringIndex or FindStringSubmatch if it is // necessary to distinguish these cases. func (re *Regexp) FindString(s string) string { - a := re.doExecute(newInputString(s), 0) + a := re.doExecute(newInputString(s), 0, 2) if a == nil { return "" } @@ -1244,7 +661,7 @@ func (re *Regexp) FindString(s string) string { // itself is at s[loc[0]:loc[1]]. // A return value of nil indicates no match. func (re *Regexp) FindStringIndex(s string) []int { - a := re.doExecute(newInputString(s), 0) + a := re.doExecute(newInputString(s), 0, 2) if a == nil { return nil } @@ -1256,7 +673,7 @@ func (re *Regexp) FindStringIndex(s string) []int { // the RuneReader. The match itself is at s[loc[0]:loc[1]]. A return // value of nil indicates no match. func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { - a := re.doExecute(newInputReader(r), 0) + a := re.doExecute(newInputReader(r), 0, 2) if a == nil { return nil } @@ -1269,13 +686,13 @@ func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { // comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatch(b []byte) [][]byte { - a := re.doExecute(newInputBytes(b), 0) + a := re.doExecute(newInputBytes(b), 0, re.prog.NumCap) if a == nil { return nil } - ret := make([][]byte, len(a)/2) + ret := make([][]byte, 1+re.numSubexp) for i := range ret { - if a[2*i] >= 0 { + if 2*i < len(a) && a[2*i] >= 0 { ret[i] = b[a[2*i]:a[2*i+1]] } } @@ -1288,7 +705,7 @@ func (re *Regexp) FindSubmatch(b []byte) [][]byte { // in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatchIndex(b []byte) []int { - return re.doExecute(newInputBytes(b), 0) + return re.pad(re.doExecute(newInputBytes(b), 0, re.prog.NumCap)) } // FindStringSubmatch returns a slice of strings holding the text of the @@ -1297,13 +714,13 @@ func (re *Regexp) FindSubmatchIndex(b []byte) []int { // package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatch(s string) []string { - a := re.doExecute(newInputString(s), 0) + a := re.doExecute(newInputString(s), 0, re.prog.NumCap) if a == nil { return nil } - ret := make([]string, len(a)/2) + ret := make([]string, 1+re.numSubexp) for i := range ret { - if a[2*i] >= 0 { + if 2*i < len(a) && a[2*i] >= 0 { ret[i] = s[a[2*i]:a[2*i+1]] } } @@ -1316,7 +733,7 @@ func (re *Regexp) FindStringSubmatch(s string) []string { // 'Index' descriptions in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatchIndex(s string) []int { - return re.doExecute(newInputString(s), 0) + return re.pad(re.doExecute(newInputString(s), 0, re.prog.NumCap)) } // FindReaderSubmatchIndex returns a slice holding the index pairs @@ -1325,7 +742,7 @@ func (re *Regexp) FindStringSubmatchIndex(s string) []int { // by the 'Submatch' and 'Index' descriptions in the package comment. A // return value of nil indicates no match. func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { - return re.doExecute(newInputReader(r), 0) + return re.pad(re.doExecute(newInputReader(r), 0, re.prog.NumCap)) } const startSize = 10 // The size at which to start a slice in the 'All' routines. diff --git a/libgo/go/exp/regexp/syntax/compile.go b/libgo/go/regexp/syntax/compile.go index 5ea2425..c415d39 100644 --- a/libgo/go/exp/regexp/syntax/compile.go +++ b/libgo/go/regexp/syntax/compile.go @@ -75,6 +75,7 @@ type compiler struct { } // Compile compiles the regexp into a program to be executed. +// The regexp should have been simplified already (returned from re.Simplify). func Compile(re *Regexp) (*Prog, os.Error) { var c compiler c.init() @@ -90,7 +91,7 @@ func (c *compiler) init() { c.inst(InstFail) } -var anyRuneNotNL = []int{0, '\n' - 1, '\n' - 1, unicode.MaxRune} +var anyRuneNotNL = []int{0, '\n' - 1, '\n' + 1, unicode.MaxRune} var anyRune = []int{0, unicode.MaxRune} func (c *compiler) compile(re *Regexp) frag { @@ -105,7 +106,7 @@ func (c *compiler) compile(re *Regexp) frag { } var f frag for j := range re.Rune { - f1 := c.rune(re.Rune[j : j+1]) + f1 := c.rune(re.Rune[j:j+1], re.Flags) if j == 0 { f = f1 } else { @@ -114,11 +115,11 @@ func (c *compiler) compile(re *Regexp) frag { } return f case OpCharClass: - return c.rune(re.Rune) + return c.rune(re.Rune, re.Flags) case OpAnyCharNotNL: - return c.rune(anyRuneNotNL) + return c.rune(anyRuneNotNL, 0) case OpAnyChar: - return c.rune(anyRune) + return c.rune(anyRune, 0) case OpBeginLine: return c.empty(EmptyBeginLine) case OpEndLine: @@ -261,9 +262,27 @@ func (c *compiler) empty(op EmptyOp) frag { return f } -func (c *compiler) rune(rune []int) frag { +func (c *compiler) rune(rune []int, flags Flags) frag { f := c.inst(InstRune) - c.p.Inst[f.i].Rune = rune + i := &c.p.Inst[f.i] + i.Rune = rune + flags &= FoldCase // only relevant flag is FoldCase + if len(rune) != 1 || unicode.SimpleFold(rune[0]) == rune[0] { + // and sometimes not even that + flags &^= FoldCase + } + i.Arg = uint32(flags) f.out = patchList(f.i << 1) + + // Special cases for exec machine. + switch { + case flags&FoldCase == 0 && (len(rune) == 1 || len(rune) == 2 && rune[0] == rune[1]): + i.Op = InstRune1 + case len(rune) == 2 && rune[0] == 0 && rune[1] == unicode.MaxRune: + i.Op = InstRuneAny + case len(rune) == 4 && rune[0] == 0 && rune[1] == '\n'-1 && rune[2] == '\n'+1 && rune[3] == unicode.MaxRune: + i.Op = InstRuneAnyNotNL + } + return f } diff --git a/libgo/go/exp/regexp/syntax/parse.go b/libgo/go/regexp/syntax/parse.go index 4eed182..7013459 100644 --- a/libgo/go/exp/regexp/syntax/parse.go +++ b/libgo/go/regexp/syntax/parse.go @@ -181,11 +181,29 @@ func (p *parser) maybeConcat(r int, flags Flags) bool { func (p *parser) newLiteral(r int, flags Flags) *Regexp { re := p.newRegexp(OpLiteral) re.Flags = flags + if flags&FoldCase != 0 { + r = minFoldRune(r) + } re.Rune0[0] = r re.Rune = re.Rune0[:1] return re } +// minFoldRune returns the minimum rune fold-equivalent to r. +func minFoldRune(r int) int { + if r < MinFold || r > MaxFold { + return r + } + min := r + r0 := r + for r = unicode.SimpleFold(r); r != r0; r = unicode.SimpleFold(r) { + if min > r { + min = r + } + } + return min +} + // literal pushes a literal regexp for the rune r on the stack // and returns that regexp. func (p *parser) literal(r int) { @@ -200,27 +218,32 @@ func (p *parser) op(op Op) *Regexp { return p.push(re) } -// repeat replaces the top stack element with itself repeated -// according to op. -func (p *parser) repeat(op Op, min, max int, opstr, t, lastRepeat string) (string, os.Error) { +// repeat replaces the top stack element with itself repeated according to op, min, max. +// before is the regexp suffix starting at the repetition operator. +// after is the regexp suffix following after the repetition operator. +// repeat returns an updated 'after' and an error, if any. +func (p *parser) repeat(op Op, min, max int, before, after, lastRepeat string) (string, os.Error) { flags := p.flags if p.flags&PerlX != 0 { - if len(t) > 0 && t[0] == '?' { - t = t[1:] + if len(after) > 0 && after[0] == '?' { + after = after[1:] flags ^= NonGreedy } if lastRepeat != "" { // In Perl it is not allowed to stack repetition operators: // a** is a syntax error, not a doubled star, and a++ means // something else entirely, which we don't support! - return "", &Error{ErrInvalidRepeatOp, lastRepeat[:len(lastRepeat)-len(t)]} + return "", &Error{ErrInvalidRepeatOp, lastRepeat[:len(lastRepeat)-len(after)]} } } n := len(p.stack) if n == 0 { - return "", &Error{ErrMissingRepeatArgument, opstr} + return "", &Error{ErrMissingRepeatArgument, before[:len(before)-len(after)]} } sub := p.stack[n-1] + if sub.Op >= opPseudo { + return "", &Error{ErrMissingRepeatArgument, before[:len(before)-len(after)]} + } re := p.newRegexp(op) re.Min = min re.Max = max @@ -228,7 +251,7 @@ func (p *parser) repeat(op Op, min, max int, opstr, t, lastRepeat string) (strin re.Sub = re.Sub0[:1] re.Sub[0] = sub p.stack[n-1] = re - return t, nil + return after, nil } // concat replaces the top of the stack (above the topmost '|' or '(') with its concatenation. @@ -419,8 +442,7 @@ func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { // used or marked for reuse, and the slice space has been reused // for out (len(out) <= start). // - // Invariant: sub[start:i] consists of regexps that all begin - // with str as modified by strflags. + // Invariant: sub[start:i] consists of regexps that all begin with ifirst. var ifirst *Regexp if i < len(sub) { ifirst = p.leadingRegexp(sub[i]) @@ -441,7 +463,6 @@ func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { } else { // Construct factored form: prefix(suffix1|suffix2|...) prefix := first - for j := start; j < i; j++ { reuse := j != start // prefix came from sub[start] sub[j] = p.removeLeadingRegexp(sub[j], reuse) @@ -605,8 +626,10 @@ func (p *parser) removeLeadingRegexp(re *Regexp, reuse bool) *Regexp { } return re } - re.Op = OpEmptyMatch - return re + if reuse { + p.reuse(re) + } + return p.newRegexp(OpEmptyMatch) } func literalRegexp(s string, flags Flags) *Regexp { @@ -704,6 +727,7 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { return nil, err } case '*', '+', '?': + before := t switch t[0] { case '*': op = OpStar @@ -712,21 +736,31 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { case '?': op = OpQuest } - if t, err = p.repeat(op, min, max, t[:1], t[1:], lastRepeat); err != nil { + after := t[1:] + if after, err = p.repeat(op, min, max, before, after, lastRepeat); err != nil { return nil, err } + repeat = before + t = after case '{': op = OpRepeat - min, max, tt, ok := p.parseRepeat(t) + before := t + min, max, after, ok := p.parseRepeat(t) if !ok { // If the repeat cannot be parsed, { is a literal. p.literal('{') t = t[1:] break } - if t, err = p.repeat(op, min, max, t[:len(t)-len(tt)], tt, lastRepeat); err != nil { + if min < 0 || min > 1000 || max > 1000 || max >= 0 && min > max { + // Numbers were too big, or max is present and min > max. + return nil, &Error{ErrInvalidRepeatSize, before[:len(before)-len(after)]} + } + if after, err = p.repeat(op, min, max, before, after, lastRepeat); err != nil { return nil, err } + repeat = before + t = after case '\\': if p.flags&PerlX != 0 && len(t) >= 2 { switch t[1] { @@ -815,12 +849,14 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { // parseRepeat parses {min} (max=min) or {min,} (max=-1) or {min,max}. // If s is not of that form, it returns ok == false. +// If s has the right form but the values are too big, it returns min == -1, ok == true. func (p *parser) parseRepeat(s string) (min, max int, rest string, ok bool) { if s == "" || s[0] != '{' { return } s = s[1:] - if min, s, ok = p.parseInt(s); !ok { + var ok1 bool + if min, s, ok1 = p.parseInt(s); !ok1 { return } if s == "" { @@ -835,8 +871,11 @@ func (p *parser) parseRepeat(s string) (min, max int, rest string, ok bool) { } if s[0] == '}' { max = -1 - } else if max, s, ok = p.parseInt(s); !ok { + } else if max, s, ok1 = p.parseInt(s); !ok1 { return + } else if max < 0 { + // parseInt found too big a number + min = -1 } } if s == "" || s[0] != '}' { @@ -981,16 +1020,22 @@ func (p *parser) parseInt(s string) (n int, rest string, ok bool) { if len(s) >= 2 && s[0] == '0' && '0' <= s[1] && s[1] <= '9' { return } + t := s for s != "" && '0' <= s[0] && s[0] <= '9' { - // Avoid overflow. - if n >= 1e8 { - return - } - n = n*10 + int(s[0]) - '0' s = s[1:] } rest = s ok = true + // Have digits, compute value. + t = t[:len(t)-len(s)] + for i := 0; i < len(t); i++ { + // Avoid overflow. + if n >= 1e8 { + n = -1 + break + } + n = n*10 + int(t[i]) - '0' + } return } @@ -1053,18 +1098,18 @@ func mergeCharClass(dst, src *Regexp) { case OpCharClass: // src is simpler, so either literal or char class if src.Op == OpLiteral { - dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + dst.Rune = appendLiteral(dst.Rune, src.Rune[0], src.Flags) } else { dst.Rune = appendClass(dst.Rune, src.Rune) } case OpLiteral: // both literal - if src.Rune[0] == dst.Rune[0] { + if src.Rune[0] == dst.Rune[0] && src.Flags == dst.Flags { break } dst.Op = OpCharClass - dst.Rune = append(dst.Rune, dst.Rune[0]) - dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + dst.Rune = appendLiteral(dst.Rune[:0], dst.Rune[0], dst.Flags) + dst.Rune = appendLiteral(dst.Rune, src.Rune[0], src.Flags) } } @@ -1125,6 +1170,8 @@ func (p *parser) parseRightParen() os.Error { if re2.Op != opLeftParen { return &Error{ErrMissingParen, p.wholeRegexp} } + // Restore flags at time of paren. + p.flags = re2.Flags if re2.Cap == 0 { // Just for grouping. p.push(re1) @@ -1330,9 +1377,18 @@ func (p *parser) appendGroup(r []int, g charGroup) []int { return r } +var anyTable = &unicode.RangeTable{ + []unicode.Range16{{0, 1<<16 - 1, 1}}, + []unicode.Range32{{1 << 16, unicode.MaxRune, 1}}, +} + // unicodeTable returns the unicode.RangeTable identified by name // and the table of additional fold-equivalent code points. func unicodeTable(name string) (*unicode.RangeTable, *unicode.RangeTable) { + // Special case: "Any" means any. + if name == "Any" { + return anyTable, anyTable + } if t := unicode.Categories[name]; t != nil { return t, unicode.FoldCategory[name] } @@ -1494,7 +1550,7 @@ func (p *parser) parseClass(s string) (rest string, err os.Error) { } } if p.flags&FoldCase == 0 { - class = appendRange(class, lo, hi) + class = AppendRange(class, lo, hi) } else { class = appendFoldedRange(class, lo, hi) } @@ -1544,8 +1600,16 @@ func cleanClass(rp *[]int) []int { return r[:w] } -// appendRange returns the result of appending the range lo-hi to the class r. -func appendRange(r []int, lo, hi int) []int { +// appendLiteral returns the result of appending the literal x to the class r. +func appendLiteral(r []int, x int, flags Flags) []int { + if flags&FoldCase != 0 { + return appendFoldedRange(r, x, x) + } + return AppendRange(r, x, x) +} + +// AppendRange returns the result of appending the range lo-hi to the class r. +func AppendRange(r []int, lo, hi int) []int { // Expand last range or next to last range if it overlaps or abuts. // Checking two ranges helps when appending case-folded // alphabets, so that one range can be expanding A-Z and the @@ -1572,39 +1636,39 @@ func appendRange(r []int, lo, hi int) []int { const ( // minimum and maximum runes involved in folding. // checked during test. - minFold = 0x0041 - maxFold = 0x1044f + MinFold = 0x0041 + MaxFold = 0x1044f ) // appendFoldedRange returns the result of appending the range lo-hi // and its case folding-equivalent runes to the class r. func appendFoldedRange(r []int, lo, hi int) []int { // Optimizations. - if lo <= minFold && hi >= maxFold { + if lo <= MinFold && hi >= MaxFold { // Range is full: folding can't add more. - return appendRange(r, lo, hi) + return AppendRange(r, lo, hi) } - if hi < minFold || lo > maxFold { + if hi < MinFold || lo > MaxFold { // Range is outside folding possibilities. - return appendRange(r, lo, hi) + return AppendRange(r, lo, hi) } - if lo < minFold { - // [lo, minFold-1] needs no folding. - r = appendRange(r, lo, minFold-1) - lo = minFold + if lo < MinFold { + // [lo, MinFold-1] needs no folding. + r = AppendRange(r, lo, MinFold-1) + lo = MinFold } - if hi > maxFold { - // [maxFold+1, hi] needs no folding. - r = appendRange(r, maxFold+1, hi) - hi = maxFold + if hi > MaxFold { + // [MaxFold+1, hi] needs no folding. + r = AppendRange(r, MaxFold+1, hi) + hi = MaxFold } - // Brute force. Depend on appendRange to coalesce ranges on the fly. + // Brute force. Depend on AppendRange to coalesce ranges on the fly. for c := lo; c <= hi; c++ { - r = appendRange(r, c, c) + r = AppendRange(r, c, c) f := unicode.SimpleFold(c) for f != c { - r = appendRange(r, f, f) + r = AppendRange(r, f, f) f = unicode.SimpleFold(f) } } @@ -1615,7 +1679,7 @@ func appendFoldedRange(r []int, lo, hi int) []int { // It assume x is clean. func appendClass(r []int, x []int) []int { for i := 0; i < len(x); i += 2 { - r = appendRange(r, x[i], x[i+1]) + r = AppendRange(r, x[i], x[i+1]) } return r } @@ -1635,12 +1699,12 @@ func appendNegatedClass(r []int, x []int) []int { for i := 0; i < len(x); i += 2 { lo, hi := x[i], x[i+1] if nextLo <= lo-1 { - r = appendRange(r, nextLo, lo-1) + r = AppendRange(r, nextLo, lo-1) } nextLo = hi + 1 } if nextLo <= unicode.MaxRune { - r = appendRange(r, nextLo, unicode.MaxRune) + r = AppendRange(r, nextLo, unicode.MaxRune) } return r } @@ -1650,21 +1714,21 @@ func appendTable(r []int, x *unicode.RangeTable) []int { for _, xr := range x.R16 { lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) if stride == 1 { - r = appendRange(r, lo, hi) + r = AppendRange(r, lo, hi) continue } for c := lo; c <= hi; c += stride { - r = appendRange(r, c, c) + r = AppendRange(r, c, c) } } for _, xr := range x.R32 { lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) if stride == 1 { - r = appendRange(r, lo, hi) + r = AppendRange(r, lo, hi) continue } for c := lo; c <= hi; c += stride { - r = appendRange(r, c, c) + r = AppendRange(r, c, c) } } return r @@ -1677,14 +1741,14 @@ func appendNegatedTable(r []int, x *unicode.RangeTable) []int { lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) if stride == 1 { if nextLo <= lo-1 { - r = appendRange(r, nextLo, lo-1) + r = AppendRange(r, nextLo, lo-1) } nextLo = hi + 1 continue } for c := lo; c <= hi; c += stride { if nextLo <= c-1 { - r = appendRange(r, nextLo, c-1) + r = AppendRange(r, nextLo, c-1) } nextLo = c + 1 } @@ -1693,20 +1757,20 @@ func appendNegatedTable(r []int, x *unicode.RangeTable) []int { lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) if stride == 1 { if nextLo <= lo-1 { - r = appendRange(r, nextLo, lo-1) + r = AppendRange(r, nextLo, lo-1) } nextLo = hi + 1 continue } for c := lo; c <= hi; c += stride { if nextLo <= c-1 { - r = appendRange(r, nextLo, c-1) + r = AppendRange(r, nextLo, c-1) } nextLo = c + 1 } } if nextLo <= unicode.MaxRune { - r = appendRange(r, nextLo, unicode.MaxRune) + r = AppendRange(r, nextLo, unicode.MaxRune) } return r } diff --git a/libgo/go/exp/regexp/syntax/parse_test.go b/libgo/go/regexp/syntax/parse_test.go index 779b9af..5d9085b 100644 --- a/libgo/go/exp/regexp/syntax/parse_test.go +++ b/libgo/go/regexp/syntax/parse_test.go @@ -2,19 +2,22 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package syntax +package syntax_test import ( + . "regexp/syntax" "bytes" "fmt" "testing" "unicode" ) -var parseTests = []struct { +type parseTest struct { Regexp string Dump string -}{ +} + +var parseTests = []parseTest{ // Base cases {`a`, `lit{a}`}, {`a.`, `cat{lit{a}dot{}}`}, @@ -38,6 +41,12 @@ var parseTests = []struct { {`a{2}?`, `nrep{2,2 lit{a}}`}, {`a{2,3}?`, `nrep{2,3 lit{a}}`}, {`a{2,}?`, `nrep{2,-1 lit{a}}`}, + // Malformed { } are treated as literals. + {`x{1001`, `str{x{1001}`}, + {`x{9876543210`, `str{x{9876543210}`}, + {`x{9876543210,`, `str{x{9876543210,}`}, + {`x{2,1`, `str{x{2,1}`}, + {`x{1,9876543210`, `str{x{1,9876543210}`}, {``, `emp{}`}, {`|`, `emp{}`}, // alt{emp{}emp{}} but got factored {`|x|`, `alt{emp{}lit{x}emp{}}`}, @@ -101,6 +110,8 @@ var parseTests = []struct { {`\p{Lu}`, mkCharClass(unicode.IsUpper)}, {`[\p{Lu}]`, mkCharClass(unicode.IsUpper)}, {`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)}, + {`\p{Any}`, `dot{}`}, + {`\p{^Any}`, `cc{}`}, // Hex, octal. {`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`}, @@ -162,14 +173,92 @@ var parseTests = []struct { // Factoring. {`abc|abd|aef|bcx|bcy`, `alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}cat{str{bc}cc{0x78-0x79}}}`}, {`ax+y|ax+z|ay+w`, `cat{lit{a}alt{cat{plus{lit{x}}cc{0x79-0x7a}}cat{plus{lit{y}}lit{w}}}}`}, + + // Bug fixes. + {`(?:.)`, `dot{}`}, + {`(?:x|(?:xa))`, `cat{lit{x}alt{emp{}lit{a}}}`}, + {`(?:.|(?:.a))`, `cat{dot{}alt{emp{}lit{a}}}`}, + {`(?:A(?:A|a))`, `cat{lit{A}litfold{A}}`}, + {`(?:A|a)`, `litfold{A}`}, + {`A|(?:A|a)`, `litfold{A}`}, + {`(?s).`, `dot{}`}, + {`(?-s).`, `dnl{}`}, + {`(?:(?:^).)`, `cat{bol{}dot{}}`}, + {`(?-s)(?:(?:^).)`, `cat{bol{}dnl{}}`}, + + // RE2 prefix_tests + {`abc|abd`, `cat{str{ab}cc{0x63-0x64}}`}, + {`a(?:b)c|abd`, `cat{str{ab}cc{0x63-0x64}}`}, + {`abc|abd|aef|bcx|bcy`, + `alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}` + + `cat{str{bc}cc{0x78-0x79}}}`}, + {`abc|x|abd`, `alt{str{abc}lit{x}str{abd}}`}, + {`(?i)abc|ABD`, `cat{strfold{AB}cc{0x43-0x44 0x63-0x64}}`}, + {`[ab]c|[ab]d`, `cat{cc{0x61-0x62}cc{0x63-0x64}}`}, + {`(?:xx|yy)c|(?:xx|yy)d`, + `cat{alt{str{xx}str{yy}}cc{0x63-0x64}}`}, + {`x{2}|x{2}[0-9]`, + `cat{rep{2,2 lit{x}}alt{emp{}cc{0x30-0x39}}}`}, + {`x{2}y|x{2}[0-9]y`, + `cat{rep{2,2 lit{x}}alt{lit{y}cat{cc{0x30-0x39}lit{y}}}}`}, } const testFlags = MatchNL | PerlX | UnicodeGroups +func TestParseSimple(t *testing.T) { + testParseDump(t, parseTests, testFlags) +} + +var foldcaseTests = []parseTest{ + {`AbCdE`, `strfold{ABCDE}`}, + {`[Aa]`, `litfold{A}`}, + {`a`, `litfold{A}`}, + + // 0x17F is an old English long s (looks like an f) and folds to s. + // 0x212A is the Kelvin symbol and folds to k. + {`A[F-g]`, `cat{litfold{A}cc{0x41-0x7a 0x17f 0x212a}}`}, // [Aa][A-z...] + {`[[:upper:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`[[:lower:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, +} + +func TestParseFoldCase(t *testing.T) { + testParseDump(t, foldcaseTests, FoldCase) +} + +var literalTests = []parseTest{ + {"(|)^$.[*+?]{5,10},\\", "str{(|)^$.[*+?]{5,10},\\}"}, +} + +func TestParseLiteral(t *testing.T) { + testParseDump(t, literalTests, Literal) +} + +var matchnlTests = []parseTest{ + {`.`, `dot{}`}, + {"\n", "lit{\n}"}, + {`[^a]`, `cc{0x0-0x60 0x62-0x10ffff}`}, + {`[a\n]`, `cc{0xa 0x61}`}, +} + +func TestParseMatchNL(t *testing.T) { + testParseDump(t, matchnlTests, MatchNL) +} + +var nomatchnlTests = []parseTest{ + {`.`, `dnl{}`}, + {"\n", "lit{\n}"}, + {`[^a]`, `cc{0x0-0x9 0xb-0x60 0x62-0x10ffff}`}, + {`[a\n]`, `cc{0xa 0x61}`}, +} + +func TestParseNoMatchNL(t *testing.T) { + testParseDump(t, nomatchnlTests, 0) +} + // Test Parse -> Dump. -func TestParseDump(t *testing.T) { - for _, tt := range parseTests { - re, err := Parse(tt.Regexp, testFlags) +func testParseDump(t *testing.T, tests []parseTest, flags Flags) { + for _, tt := range tests { + re, err := Parse(tt.Regexp, flags) if err != nil { t.Errorf("Parse(%#q): %v", tt.Regexp, err) continue @@ -324,13 +413,13 @@ func TestFoldConstants(t *testing.T) { if unicode.SimpleFold(i) == i { continue } - if last == -1 && minFold != i { - t.Errorf("minFold=%#U should be %#U", minFold, i) + if last == -1 && MinFold != i { + t.Errorf("MinFold=%#U should be %#U", MinFold, i) } last = i } - if maxFold != last { - t.Errorf("maxFold=%#U should be %#U", maxFold, last) + if MaxFold != last { + t.Errorf("MaxFold=%#U should be %#U", MaxFold, last) } } @@ -341,10 +430,123 @@ func TestAppendRangeCollapse(t *testing.T) { // Note that we are not calling cleanClass. var r []int for i := 'A'; i <= 'Z'; i++ { - r = appendRange(r, i, i) - r = appendRange(r, i+'a'-'A', i+'a'-'A') + r = AppendRange(r, i, i) + r = AppendRange(r, i+'a'-'A', i+'a'-'A') } if string(r) != "AZaz" { - t.Errorf("appendRange interlaced A-Z a-z = %s, want AZaz", string(r)) + t.Errorf("AppendRange interlaced A-Z a-z = %s, want AZaz", string(r)) + } +} + +var invalidRegexps = []string{ + `(`, + `)`, + `(a`, + `(a|b|`, + `(a|b`, + `[a-z`, + `([a-z)`, + `x{1001}`, + `x{9876543210}`, + `x{2,1}`, + `x{1,9876543210}`, + "\xff", // Invalid UTF-8 + "[\xff]", + "[\\\xff]", + "\\\xff", + `(?P<name>a`, + `(?P<name>`, + `(?P<name`, + `(?P<x y>a)`, + `(?P<>a)`, + `[a-Z]`, + `(?i)[a-Z]`, + `a{100000}`, + `a{100000,}`, +} + +var onlyPerl = []string{ + `[a-b-c]`, + `\Qabc\E`, + `\Q*+?{[\E`, + `\Q\\E`, + `\Q\\\E`, + `\Q\\\\E`, + `\Q\\\\\E`, + `(?:a)`, + `(?P<name>a)`, +} + +var onlyPOSIX = []string{ + "a++", + "a**", + "a?*", + "a+*", + "a{1}*", + ".{1}{2}.{3}", +} + +func TestParseInvalidRegexps(t *testing.T) { + for _, regexp := range invalidRegexps { + if re, err := Parse(regexp, Perl); err == nil { + t.Errorf("Parse(%#q, Perl) = %s, should have failed", regexp, dump(re)) + } + if re, err := Parse(regexp, POSIX); err == nil { + t.Errorf("Parse(%#q, POSIX) = %s, should have failed", regexp, dump(re)) + } + } + for _, regexp := range onlyPerl { + if _, err := Parse(regexp, Perl); err != nil { + t.Errorf("Parse(%#q, Perl): %v", regexp, err) + } + if re, err := Parse(regexp, POSIX); err == nil { + t.Errorf("Parse(%#q, POSIX) = %s, should have failed", regexp, dump(re)) + } + } + for _, regexp := range onlyPOSIX { + if re, err := Parse(regexp, Perl); err == nil { + t.Errorf("Parse(%#q, Perl) = %s, should have failed", regexp, dump(re)) + } + if _, err := Parse(regexp, POSIX); err != nil { + t.Errorf("Parse(%#q, POSIX): %v", regexp, err) + } + } +} + +func TestToStringEquivalentParse(t *testing.T) { + for _, tt := range parseTests { + re, err := Parse(tt.Regexp, testFlags) + if err != nil { + t.Errorf("Parse(%#q): %v", tt.Regexp, err) + continue + } + d := dump(re) + if d != tt.Dump { + t.Errorf("Parse(%#q).Dump() = %#q want %#q", tt.Regexp, d, tt.Dump) + continue + } + + s := re.String() + if s != tt.Regexp { + // If ToString didn't return the original regexp, + // it must have found one with fewer parens. + // Unfortunately we can't check the length here, because + // ToString produces "\\{" for a literal brace, + // but "{" is a shorter equivalent in some contexts. + nre, err := Parse(s, testFlags) + if err != nil { + t.Errorf("Parse(%#q.String() = %#q): %v", tt.Regexp, t, err) + continue + } + nd := dump(nre) + if d != nd { + t.Errorf("Parse(%#q) -> %#q; %#q vs %#q", tt.Regexp, s, d, nd) + } + + ns := nre.String() + if s != ns { + t.Errorf("Parse(%#q) -> %#q -> %#q", tt.Regexp, s, ns) + } + } } } diff --git a/libgo/go/exp/regexp/syntax/perl_groups.go b/libgo/go/regexp/syntax/perl_groups.go index 05b392c..05b392c 100644 --- a/libgo/go/exp/regexp/syntax/perl_groups.go +++ b/libgo/go/regexp/syntax/perl_groups.go diff --git a/libgo/go/exp/regexp/syntax/prog.go b/libgo/go/regexp/syntax/prog.go index bf85b72..ced45da 100644 --- a/libgo/go/exp/regexp/syntax/prog.go +++ b/libgo/go/regexp/syntax/prog.go @@ -3,6 +3,7 @@ package syntax import ( "bytes" "strconv" + "unicode" ) // Compiled program. @@ -27,6 +28,9 @@ const ( InstFail InstNop InstRune + InstRune1 + InstRuneAny + InstRuneAnyNotNL ) // An EmptyOp specifies a kind or mixture of zero-width assertions. @@ -41,6 +45,41 @@ const ( EmptyNoWordBoundary ) +// EmptyOpContext returns the zero-width assertions +// satisfied at the position between the runes r1 and r2. +// Passing r1 == -1 indicates that the position is +// at the beginning of the text. +// Passing r2 == -1 indicates that the position is +// at the end of the text. +func EmptyOpContext(r1, r2 int) EmptyOp { + var op EmptyOp + if r1 < 0 { + op |= EmptyBeginText | EmptyBeginLine + } + if r1 == '\n' { + op |= EmptyBeginLine + } + if r2 < 0 { + op |= EmptyEndText | EmptyEndLine + } + if r2 == '\n' { + op |= EmptyEndLine + } + if IsWordChar(r1) != IsWordChar(r2) { + op |= EmptyWordBoundary + } else { + op |= EmptyNoWordBoundary + } + return op +} + +// IsWordChar reports whether r is consider a ``word character'' +// during the evaluation of the \b and \B zero-width assertions. +// These assertions are ASCII-only: the word characters are [A-Za-z0-9_]. +func IsWordChar(r int) bool { + return 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' || '0' <= r && r <= '9' || r == '_' +} + // An Inst is a single instruction in a regular expression program. type Inst struct { Op InstOp @@ -66,6 +105,16 @@ func (p *Prog) skipNop(pc uint32) *Inst { return i } +// op returns i.Op but merges all the Rune special cases into InstRune +func (i *Inst) op() InstOp { + op := i.Op + switch op { + case InstRune1, InstRuneAny, InstRuneAnyNotNL: + op = InstRune + } + return op +} + // Prefix returns a literal string that all matches for the // regexp must start with. Complete is true if the prefix // is the entire match. @@ -73,13 +122,13 @@ func (p *Prog) Prefix() (prefix string, complete bool) { i := p.skipNop(uint32(p.Start)) // Avoid allocation of buffer if prefix is empty. - if i.Op != InstRune || len(i.Rune) != 1 { + if i.op() != InstRune || len(i.Rune) != 1 { return "", i.Op == InstMatch } // Have prefix; gather characters. var buf bytes.Buffer - for i.Op == InstRune && len(i.Rune) == 1 { + for i.op() == InstRune && len(i.Rune) == 1 && Flags(i.Arg)&FoldCase == 0 { buf.WriteRune(i.Rune[0]) i = p.skipNop(i.Out) } @@ -116,9 +165,19 @@ func (i *Inst) MatchRune(r int) bool { rune := i.Rune // Special case: single-rune slice is from literal string, not char class. - // TODO: Case folding. if len(rune) == 1 { - return r == rune[0] + r0 := rune[0] + if r == r0 { + return true + } + if Flags(i.Arg)&FoldCase != 0 { + for r1 := unicode.SimpleFold(r0); r1 != r0; r1 = unicode.SimpleFold(r1) { + if r == r1 { + return true + } + } + } + return false } // Peek at the first few pairs. @@ -232,6 +291,16 @@ func dumpInst(b *bytes.Buffer, i *Inst) { // shouldn't happen bw(b, "rune <nil>") } - bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune)), " -> ", u32(i.Out)) + bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune))) + if Flags(i.Arg)&FoldCase != 0 { + bw(b, "/i") + } + bw(b, " -> ", u32(i.Out)) + case InstRune1: + bw(b, "rune1 ", strconv.QuoteToASCII(string(i.Rune)), " -> ", u32(i.Out)) + case InstRuneAny: + bw(b, "any -> ", u32(i.Out)) + case InstRuneAnyNotNL: + bw(b, "anynotnl -> ", u32(i.Out)) } } diff --git a/libgo/go/exp/regexp/syntax/prog_test.go b/libgo/go/regexp/syntax/prog_test.go index 7be4281..70959f6 100644 --- a/libgo/go/exp/regexp/syntax/prog_test.go +++ b/libgo/go/regexp/syntax/prog_test.go @@ -1,6 +1,7 @@ -package syntax +package syntax_test import ( + . "regexp/syntax" "testing" ) @@ -9,7 +10,7 @@ var compileTests = []struct { Prog string }{ {"a", ` 0 fail - 1* rune "a" -> 2 + 1* rune1 "a" -> 2 2 match `}, {"[A-M][n-z]", ` 0 fail @@ -22,61 +23,71 @@ var compileTests = []struct { 2 match `}, {"a?", ` 0 fail - 1 rune "a" -> 3 + 1 rune1 "a" -> 3 2* alt -> 1, 3 3 match `}, {"a??", ` 0 fail - 1 rune "a" -> 3 + 1 rune1 "a" -> 3 2* alt -> 3, 1 3 match `}, {"a+", ` 0 fail - 1* rune "a" -> 2 + 1* rune1 "a" -> 2 2 alt -> 1, 3 3 match `}, {"a+?", ` 0 fail - 1* rune "a" -> 2 + 1* rune1 "a" -> 2 2 alt -> 3, 1 3 match `}, {"a*", ` 0 fail - 1 rune "a" -> 2 + 1 rune1 "a" -> 2 2* alt -> 1, 3 3 match `}, {"a*?", ` 0 fail - 1 rune "a" -> 2 + 1 rune1 "a" -> 2 2* alt -> 3, 1 3 match `}, {"a+b+", ` 0 fail - 1* rune "a" -> 2 + 1* rune1 "a" -> 2 2 alt -> 1, 3 - 3 rune "b" -> 4 + 3 rune1 "b" -> 4 4 alt -> 3, 5 5 match `}, {"(a+)(b+)", ` 0 fail 1* cap 2 -> 2 - 2 rune "a" -> 3 + 2 rune1 "a" -> 3 3 alt -> 2, 4 4 cap 3 -> 5 5 cap 4 -> 6 - 6 rune "b" -> 7 + 6 rune1 "b" -> 7 7 alt -> 6, 8 8 cap 5 -> 9 9 match `}, {"a+|b+", ` 0 fail - 1 rune "a" -> 2 + 1 rune1 "a" -> 2 2 alt -> 1, 6 - 3 rune "b" -> 4 + 3 rune1 "b" -> 4 4 alt -> 3, 6 5* alt -> 1, 3 6 match `}, + {"A[Aa]", ` 0 fail + 1* rune1 "A" -> 2 + 2 rune "A"/i -> 3 + 3 match +`}, + {"(?:(?:^).)", ` 0 fail + 1* empty 4 -> 2 + 2 anynotnl -> 3 + 3 match +`}, } func TestCompile(t *testing.T) { diff --git a/libgo/go/exp/regexp/syntax/regexp.go b/libgo/go/regexp/syntax/regexp.go index 00a4add..033848d 100644 --- a/libgo/go/exp/regexp/syntax/regexp.go +++ b/libgo/go/regexp/syntax/regexp.go @@ -164,9 +164,9 @@ func writeRegexp(b *bytes.Buffer, re *Regexp) { } b.WriteRune(']') case OpAnyCharNotNL: - b.WriteString(`[^\n]`) + b.WriteString(`(?-s:.)`) case OpAnyChar: - b.WriteRune('.') + b.WriteString(`(?s:.)`) case OpBeginLine: b.WriteRune('^') case OpEndLine: @@ -174,7 +174,11 @@ func writeRegexp(b *bytes.Buffer, re *Regexp) { case OpBeginText: b.WriteString(`\A`) case OpEndText: - b.WriteString(`\z`) + if re.Flags&WasDollar != 0 { + b.WriteString(`(?-m:$)`) + } else { + b.WriteString(`\z`) + } case OpWordBoundary: b.WriteString(`\b`) case OpNoWordBoundary: @@ -192,7 +196,7 @@ func writeRegexp(b *bytes.Buffer, re *Regexp) { } b.WriteRune(')') case OpStar, OpPlus, OpQuest, OpRepeat: - if sub := re.Sub[0]; sub.Op > OpCapture { + if sub := re.Sub[0]; sub.Op > OpCapture || sub.Op == OpLiteral && len(sub.Rune) > 1 { b.WriteString(`(?:`) writeRegexp(b, sub) b.WriteString(`)`) @@ -217,6 +221,9 @@ func writeRegexp(b *bytes.Buffer, re *Regexp) { } b.WriteRune('}') } + if re.Flags&NonGreedy != 0 { + b.WriteRune('?') + } case OpConcat: for _, sub := range re.Sub { if sub.Op == OpAlternate { @@ -282,3 +289,17 @@ func escape(b *bytes.Buffer, r int, force bool) { b.WriteString(`}`) } } + +// MaxCap walks the regexp to find the maximum capture index. +func (re *Regexp) MaxCap() int { + m := 0 + if re.Op == OpCapture { + m = re.Cap + } + for _, sub := range re.Sub { + if n := sub.MaxCap(); m < n { + m = n + } + } + return m +} diff --git a/libgo/go/exp/regexp/syntax/simplify.go b/libgo/go/regexp/syntax/simplify.go index 7239041..7239041 100644 --- a/libgo/go/exp/regexp/syntax/simplify.go +++ b/libgo/go/regexp/syntax/simplify.go diff --git a/libgo/go/exp/regexp/syntax/simplify_test.go b/libgo/go/regexp/syntax/simplify_test.go index c8cec21..92a9d3d 100644 --- a/libgo/go/exp/regexp/syntax/simplify_test.go +++ b/libgo/go/regexp/syntax/simplify_test.go @@ -2,8 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package syntax +package syntax_test +import . "regexp/syntax" import "testing" var simplifyTests = []struct { @@ -18,7 +19,7 @@ var simplifyTests = []struct { {`(ab)*`, `(ab)*`}, {`(ab)+`, `(ab)+`}, {`(ab)?`, `(ab)?`}, - {`.`, `.`}, + {`.`, `(?s:.)`}, {`^`, `^`}, {`$`, `$`}, {`[ac]`, `[ac]`}, @@ -97,22 +98,22 @@ var simplifyTests = []struct { {`[^[:cntrl:][:^cntrl:]]`, `[^\x00-\x{10FFFF}]`}, // Full character classes - {`[[:cntrl:][:^cntrl:]]`, `.`}, + {`[[:cntrl:][:^cntrl:]]`, `(?s:.)`}, // Unicode case folding. {`(?i)A`, `(?i:A)`}, - {`(?i)a`, `(?i:a)`}, + {`(?i)a`, `(?i:A)`}, {`(?i)[A]`, `(?i:A)`}, {`(?i)[a]`, `(?i:A)`}, {`(?i)K`, `(?i:K)`}, - {`(?i)k`, `(?i:k)`}, - {`(?i)\x{212a}`, "(?i:\u212A)"}, + {`(?i)k`, `(?i:K)`}, + {`(?i)\x{212a}`, "(?i:K)"}, {`(?i)[K]`, "[Kk\u212A]"}, {`(?i)[k]`, "[Kk\u212A]"}, {`(?i)[\x{212a}]`, "[Kk\u212A]"}, {`(?i)[a-z]`, "[A-Za-z\u017F\u212A]"}, {`(?i)[\x00-\x{FFFD}]`, "[\\x00-\uFFFD]"}, - {`(?i)[\x00-\x{10FFFF}]`, `.`}, + {`(?i)[\x00-\x{10FFFF}]`, `(?s:.)`}, // Empty string as a regular expression. // The empty string must be preserved inside parens in order diff --git a/libgo/go/regexp/testdata/README b/libgo/go/regexp/testdata/README new file mode 100644 index 0000000..b1b301b --- /dev/null +++ b/libgo/go/regexp/testdata/README @@ -0,0 +1,23 @@ +AT&T POSIX Test Files +See textregex.c for copyright + license. + +testregex.c http://www2.research.att.com/~gsf/testregex/testregex.c +basic.dat http://www2.research.att.com/~gsf/testregex/basic.dat +nullsubexpr.dat http://www2.research.att.com/~gsf/testregex/nullsubexpr.dat +repetition.dat http://www2.research.att.com/~gsf/testregex/repetition.dat + +The test data has been edited to reflect RE2/Go differences: + * In a star of a possibly empty match like (a*)* matching x, + the no match case runs the starred subexpression zero times, + not once. This is consistent with (a*)* matching a, which + runs the starred subexpression one time, not twice. + * The submatch choice is first match, not the POSIX rule. + +Such changes are marked with 'RE2/Go'. + + +RE2 Test Files + +re2-exhaustive.txt.bz2 and re2-search.txt are built by running +'make log' in the RE2 distribution. http://code.google.com/p/re2/. +The exhaustive file is compressed because it is huge. diff --git a/libgo/go/regexp/testdata/basic.dat b/libgo/go/regexp/testdata/basic.dat new file mode 100644 index 0000000..7859290b --- /dev/null +++ b/libgo/go/regexp/testdata/basic.dat @@ -0,0 +1,221 @@ +NOTE all standard compliant implementations should pass these : 2002-05-31 + +BE abracadabra$ abracadabracadabra (7,18) +BE a...b abababbb (2,7) +BE XXXXXX ..XXXXXX (2,8) +E \) () (1,2) +BE a] a]a (0,2) +B } } (0,1) +E \} } (0,1) +BE \] ] (0,1) +B ] ] (0,1) +E ] ] (0,1) +B { { (0,1) +B } } (0,1) +BE ^a ax (0,1) +BE \^a a^a (1,3) +BE a\^ a^ (0,2) +BE a$ aa (1,2) +BE a\$ a$ (0,2) +BE ^$ NULL (0,0) +E $^ NULL (0,0) +E a($) aa (1,2)(2,2) +E a*(^a) aa (0,1)(0,1) +E (..)*(...)* a (0,0) +E (..)*(...)* abcd (0,4)(2,4) +E (ab|a)(bc|c) abc (0,3)(0,2)(2,3) +E (ab)c|abc abc (0,3)(0,2) +E a{0}b ab (1,2) +E (a*)(b?)(b+)b{3} aaabbbbbbb (0,10)(0,3)(3,4)(4,7) +E (a*)(b{0,1})(b{1,})b{3} aaabbbbbbb (0,10)(0,3)(3,4)(4,7) +E a{9876543210} NULL BADBR +E ((a|a)|a) a (0,1)(0,1)(0,1) +E (a*)(a|aa) aaaa (0,4)(0,3)(3,4) +E a*(a.|aa) aaaa (0,4)(2,4) +E a(b)|c(d)|a(e)f aef (0,3)(?,?)(?,?)(1,2) +E (a|b)?.* b (0,1)(0,1) +E (a|b)c|a(b|c) ac (0,2)(0,1) +E (a|b)c|a(b|c) ab (0,2)(?,?)(1,2) +E (a|b)*c|(a|ab)*c abc (0,3)(1,2) +E (a|b)*c|(a|ab)*c xc (1,2) +E (.a|.b).*|.*(.a|.b) xa (0,2)(0,2) +E a?(ab|ba)ab abab (0,4)(0,2) +E a?(ac{0}b|ba)ab abab (0,4)(0,2) +E ab|abab abbabab (0,2) +E aba|bab|bba baaabbbaba (5,8) +E aba|bab baaabbbaba (6,9) +E (aa|aaa)*|(a|aaaaa) aa (0,2)(0,2) +E (a.|.a.)*|(a|.a...) aa (0,2)(0,2) +E ab|a xabc (1,3) +E ab|a xxabc (2,4) +Ei (Ab|cD)* aBcD (0,4)(2,4) +BE [^-] --a (2,3) +BE [a-]* --a (0,3) +BE [a-m-]* --amoma-- (0,4) +E :::1:::0:|:::1:1:0: :::0:::1:::1:::0: (8,17) +E :::1:::0:|:::1:1:1: :::0:::1:::1:::0: (8,17) +{E [[:upper:]] A (0,1) [[<element>]] not supported +E [[:lower:]]+ `az{ (1,3) +E [[:upper:]]+ @AZ[ (1,3) +# No collation in Go +#BE [[-]] [[-]] (2,4) +#BE [[.NIL.]] NULL ECOLLATE +#BE [[=aleph=]] NULL ECOLLATE +} +BE$ \n \n (0,1) +BEn$ \n \n (0,1) +BE$ [^a] \n (0,1) +BE$ \na \na (0,2) +E (a)(b)(c) abc (0,3)(0,1)(1,2)(2,3) +BE xxx xxx (0,3) +E1 (^|[ (,;])((([Ff]eb[^ ]* *|0*2/|\* */?)0*[6-7]))([^0-9]|$) feb 6, (0,6) +E1 (^|[ (,;])((([Ff]eb[^ ]* *|0*2/|\* */?)0*[6-7]))([^0-9]|$) 2/7 (0,3) +E1 (^|[ (,;])((([Ff]eb[^ ]* *|0*2/|\* */?)0*[6-7]))([^0-9]|$) feb 1,Feb 6 (5,11) +E3 ((((((((((((((((((((((((((((((x)))))))))))))))))))))))))))))) x (0,1)(0,1)(0,1) +E3 ((((((((((((((((((((((((((((((x))))))))))))))))))))))))))))))* xx (0,2)(1,2)(1,2) +E a?(ab|ba)* ababababababababababababababababababababababababababababababababababababababababa (0,81)(79,81) +E abaa|abbaa|abbbaa|abbbbaa ababbabbbabbbabbbbabbbbaa (18,25) +E abaa|abbaa|abbbaa|abbbbaa ababbabbbabbbabbbbabaa (18,22) +E aaac|aabc|abac|abbc|baac|babc|bbac|bbbc baaabbbabac (7,11) +BE$ .* \x01\xff (0,2) +E aaaa|bbbb|cccc|ddddd|eeeeee|fffffff|gggg|hhhh|iiiii|jjjjj|kkkkk|llll XaaaXbbbXcccXdddXeeeXfffXgggXhhhXiiiXjjjXkkkXlllXcbaXaaaa (53,57) +L aaaa\nbbbb\ncccc\nddddd\neeeeee\nfffffff\ngggg\nhhhh\niiiii\njjjjj\nkkkkk\nllll XaaaXbbbXcccXdddXeeeXfffXgggXhhhXiiiXjjjXkkkXlllXcbaXaaaa NOMATCH +E a*a*a*a*a*b aaaaaaaaab (0,10) +BE ^ NULL (0,0) +BE $ NULL (0,0) +BE ^$ NULL (0,0) +BE ^a$ a (0,1) +BE abc abc (0,3) +BE abc xabcy (1,4) +BE abc ababc (2,5) +BE ab*c abc (0,3) +BE ab*bc abc (0,3) +BE ab*bc abbc (0,4) +BE ab*bc abbbbc (0,6) +E ab+bc abbc (0,4) +E ab+bc abbbbc (0,6) +E ab?bc abbc (0,4) +E ab?bc abc (0,3) +E ab?c abc (0,3) +BE ^abc$ abc (0,3) +BE ^abc abcc (0,3) +BE abc$ aabc (1,4) +BE ^ abc (0,0) +BE $ abc (3,3) +BE a.c abc (0,3) +BE a.c axc (0,3) +BE a.*c axyzc (0,5) +BE a[bc]d abd (0,3) +BE a[b-d]e ace (0,3) +BE a[b-d] aac (1,3) +BE a[-b] a- (0,2) +BE a[b-] a- (0,2) +BE a] a] (0,2) +BE a[]]b a]b (0,3) +BE a[^bc]d aed (0,3) +BE a[^-b]c adc (0,3) +BE a[^]b]c adc (0,3) +E ab|cd abc (0,2) +E ab|cd abcd (0,2) +E a\(b a(b (0,3) +E a\(*b ab (0,2) +E a\(*b a((b (0,4) +E ((a)) abc (0,1)(0,1)(0,1) +E (a)b(c) abc (0,3)(0,1)(2,3) +E a+b+c aabbabc (4,7) +E a* aaa (0,3) +#E (a*)* - (0,0)(0,0) +E (a*)* - (0,0)(?,?) RE2/Go +E (a*)+ - (0,0)(0,0) +#E (a*|b)* - (0,0)(0,0) +E (a*|b)* - (0,0)(?,?) RE2/Go +E (a+|b)* ab (0,2)(1,2) +E (a+|b)+ ab (0,2)(1,2) +E (a+|b)? ab (0,1)(0,1) +BE [^ab]* cde (0,3) +#E (^)* - (0,0)(0,0) +E (^)* - (0,0)(?,?) RE2/Go +BE a* NULL (0,0) +E ([abc])*d abbbcd (0,6)(4,5) +E ([abc])*bcd abcd (0,4)(0,1) +E a|b|c|d|e e (0,1) +E (a|b|c|d|e)f ef (0,2)(0,1) +#E ((a*|b))* - (0,0)(0,0)(0,0) +E ((a*|b))* - (0,0)(?,?)(?,?) RE2/Go +BE abcd*efg abcdefg (0,7) +BE ab* xabyabbbz (1,3) +BE ab* xayabbbz (1,2) +E (ab|cd)e abcde (2,5)(2,4) +BE [abhgefdc]ij hij (0,3) +E (a|b)c*d abcd (1,4)(1,2) +E (ab|ab*)bc abc (0,3)(0,1) +E a([bc]*)c* abc (0,3)(1,3) +E a([bc]*)(c*d) abcd (0,4)(1,3)(3,4) +E a([bc]+)(c*d) abcd (0,4)(1,3)(3,4) +E a([bc]*)(c+d) abcd (0,4)(1,2)(2,4) +E a[bcd]*dcdcde adcdcde (0,7) +E (ab|a)b*c abc (0,3)(0,2) +E ((a)(b)c)(d) abcd (0,4)(0,3)(0,1)(1,2)(3,4) +BE [A-Za-z_][A-Za-z0-9_]* alpha (0,5) +E ^a(bc+|b[eh])g|.h$ abh (1,3) +E (bc+d$|ef*g.|h?i(j|k)) effgz (0,5)(0,5) +E (bc+d$|ef*g.|h?i(j|k)) ij (0,2)(0,2)(1,2) +E (bc+d$|ef*g.|h?i(j|k)) reffgz (1,6)(1,6) +E (((((((((a))))))))) a (0,1)(0,1)(0,1)(0,1)(0,1)(0,1)(0,1)(0,1)(0,1)(0,1) +BE multiple words multiple words yeah (0,14) +E (.*)c(.*) abcde (0,5)(0,2)(3,5) +BE abcd abcd (0,4) +E a(bc)d abcd (0,4)(1,3) +E a[-]?c ac (0,3) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Qaddafi (0,15)(?,?)(10,12) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Mo'ammar Gadhafi (0,16)(?,?)(11,13) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Kaddafi (0,15)(?,?)(10,12) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Qadhafi (0,15)(?,?)(10,12) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Gadafi (0,14)(?,?)(10,11) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Mu'ammar Qadafi (0,15)(?,?)(11,12) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Moamar Gaddafi (0,14)(?,?)(9,11) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Mu'ammar Qadhdhafi (0,18)(?,?)(13,15) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Khaddafi (0,16)(?,?)(11,13) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Ghaddafy (0,16)(?,?)(11,13) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Ghadafi (0,15)(?,?)(11,12) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Ghaddafi (0,16)(?,?)(11,13) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muamar Kaddafi (0,14)(?,?)(9,11) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Quathafi (0,16)(?,?)(11,13) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Muammar Gheddafi (0,16)(?,?)(11,13) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Moammar Khadafy (0,15)(?,?)(11,12) +E M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy] Moammar Qudhafi (0,15)(?,?)(10,12) +E a+(b|c)*d+ aabcdd (0,6)(3,4) +E ^.+$ vivi (0,4) +E ^(.+)$ vivi (0,4)(0,4) +E ^([^!.]+).att.com!(.+)$ gryphon.att.com!eby (0,19)(0,7)(16,19) +E ^([^!]+!)?([^!]+)$ bas (0,3)(?,?)(0,3) +E ^([^!]+!)?([^!]+)$ bar!bas (0,7)(0,4)(4,7) +E ^([^!]+!)?([^!]+)$ foo!bas (0,7)(0,4)(4,7) +E ^.+!([^!]+!)([^!]+)$ foo!bar!bas (0,11)(4,8)(8,11) +E ((foo)|(bar))!bas bar!bas (0,7)(0,3)(?,?)(0,3) +E ((foo)|(bar))!bas foo!bar!bas (4,11)(4,7)(?,?)(4,7) +E ((foo)|(bar))!bas foo!bas (0,7)(0,3)(0,3) +E ((foo)|bar)!bas bar!bas (0,7)(0,3) +E ((foo)|bar)!bas foo!bar!bas (4,11)(4,7) +E ((foo)|bar)!bas foo!bas (0,7)(0,3)(0,3) +E (foo|(bar))!bas bar!bas (0,7)(0,3)(0,3) +E (foo|(bar))!bas foo!bar!bas (4,11)(4,7)(4,7) +E (foo|(bar))!bas foo!bas (0,7)(0,3) +E (foo|bar)!bas bar!bas (0,7)(0,3) +E (foo|bar)!bas foo!bar!bas (4,11)(4,7) +E (foo|bar)!bas foo!bas (0,7)(0,3) +E ^(([^!]+!)?([^!]+)|.+!([^!]+!)([^!]+))$ foo!bar!bas (0,11)(0,11)(?,?)(?,?)(4,8)(8,11) +E ^([^!]+!)?([^!]+)$|^.+!([^!]+!)([^!]+)$ bas (0,3)(?,?)(0,3) +E ^([^!]+!)?([^!]+)$|^.+!([^!]+!)([^!]+)$ bar!bas (0,7)(0,4)(4,7) +E ^([^!]+!)?([^!]+)$|^.+!([^!]+!)([^!]+)$ foo!bar!bas (0,11)(?,?)(?,?)(4,8)(8,11) +E ^([^!]+!)?([^!]+)$|^.+!([^!]+!)([^!]+)$ foo!bas (0,7)(0,4)(4,7) +E ^(([^!]+!)?([^!]+)|.+!([^!]+!)([^!]+))$ bas (0,3)(0,3)(?,?)(0,3) +E ^(([^!]+!)?([^!]+)|.+!([^!]+!)([^!]+))$ bar!bas (0,7)(0,7)(0,4)(4,7) +E ^(([^!]+!)?([^!]+)|.+!([^!]+!)([^!]+))$ foo!bar!bas (0,11)(0,11)(?,?)(?,?)(4,8)(8,11) +E ^(([^!]+!)?([^!]+)|.+!([^!]+!)([^!]+))$ foo!bas (0,7)(0,7)(0,4)(4,7) +E .*(/XXX).* /XXX (0,4)(0,4) +E .*(\\XXX).* \XXX (0,4)(0,4) +E \\XXX \XXX (0,4) +E .*(/000).* /000 (0,4)(0,4) +E .*(\\000).* \000 (0,4)(0,4) +E \\000 \000 (0,4) diff --git a/libgo/go/regexp/testdata/nullsubexpr.dat b/libgo/go/regexp/testdata/nullsubexpr.dat new file mode 100644 index 0000000..2e18fbb --- /dev/null +++ b/libgo/go/regexp/testdata/nullsubexpr.dat @@ -0,0 +1,79 @@ +NOTE null subexpression matches : 2002-06-06 + +E (a*)* a (0,1)(0,1) +#E SAME x (0,0)(0,0) +E SAME x (0,0)(?,?) RE2/Go +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaax (0,6)(0,6) +E (a*)+ a (0,1)(0,1) +E SAME x (0,0)(0,0) +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaax (0,6)(0,6) +E (a+)* a (0,1)(0,1) +E SAME x (0,0) +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaax (0,6)(0,6) +E (a+)+ a (0,1)(0,1) +E SAME x NOMATCH +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaax (0,6)(0,6) + +E ([a]*)* a (0,1)(0,1) +#E SAME x (0,0)(0,0) +E SAME x (0,0)(?,?) RE2/Go +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaax (0,6)(0,6) +E ([a]*)+ a (0,1)(0,1) +E SAME x (0,0)(0,0) +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaax (0,6)(0,6) +E ([^b]*)* a (0,1)(0,1) +#E SAME b (0,0)(0,0) +E SAME b (0,0)(?,?) RE2/Go +E SAME aaaaaa (0,6)(0,6) +E SAME aaaaaab (0,6)(0,6) +E ([ab]*)* a (0,1)(0,1) +E SAME aaaaaa (0,6)(0,6) +E SAME ababab (0,6)(0,6) +E SAME bababa (0,6)(0,6) +E SAME b (0,1)(0,1) +E SAME bbbbbb (0,6)(0,6) +E SAME aaaabcde (0,5)(0,5) +E ([^a]*)* b (0,1)(0,1) +E SAME bbbbbb (0,6)(0,6) +#E SAME aaaaaa (0,0)(0,0) +E SAME aaaaaa (0,0)(?,?) RE2/Go +E ([^ab]*)* ccccxx (0,6)(0,6) +#E SAME ababab (0,0)(0,0) +E SAME ababab (0,0)(?,?) RE2/Go + +E ((z)+|a)* zabcde (0,2)(1,2) + +#{E a+? aaaaaa (0,1) no *? +? mimimal match ops +#E (a) aaa (0,1)(0,1) +#E (a*?) aaa (0,0)(0,0) +#E (a)*? aaa (0,0) +#E (a*?)*? aaa (0,0) +#} + +B \(a*\)*\(x\) x (0,1)(0,0)(0,1) +B \(a*\)*\(x\) ax (0,2)(0,1)(1,2) +B \(a*\)*\(x\) axa (0,2)(0,1)(1,2) +B \(a*\)*\(x\)\(\1\) x (0,1)(0,0)(0,1)(1,1) +B \(a*\)*\(x\)\(\1\) ax (0,2)(1,1)(1,2)(2,2) +B \(a*\)*\(x\)\(\1\) axa (0,3)(0,1)(1,2)(2,3) +B \(a*\)*\(x\)\(\1\)\(x\) axax (0,4)(0,1)(1,2)(2,3)(3,4) +B \(a*\)*\(x\)\(\1\)\(x\) axxa (0,3)(1,1)(1,2)(2,2)(2,3) + +#E (a*)*(x) x (0,1)(0,0)(0,1) +E (a*)*(x) x (0,1)(?,?)(0,1) RE2/Go +E (a*)*(x) ax (0,2)(0,1)(1,2) +E (a*)*(x) axa (0,2)(0,1)(1,2) + +E (a*)+(x) x (0,1)(0,0)(0,1) +E (a*)+(x) ax (0,2)(0,1)(1,2) +E (a*)+(x) axa (0,2)(0,1)(1,2) + +E (a*){2}(x) x (0,1)(0,0)(0,1) +E (a*){2}(x) ax (0,2)(1,1)(1,2) +E (a*){2}(x) axa (0,2)(1,1)(1,2) diff --git a/libgo/go/regexp/testdata/re2-search.txt b/libgo/go/regexp/testdata/re2-search.txt new file mode 100644 index 0000000..f648e55 --- /dev/null +++ b/libgo/go/regexp/testdata/re2-search.txt @@ -0,0 +1,3667 @@ +# RE2 basic search tests built by make log +# Thu Sep 8 13:43:43 EDT 2011 +Regexp.SearchTests +strings +"" +"a" +regexps +"a" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:a)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:a)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:a)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"zyzzyva" +regexps +"a" +-;-;-;- +-;6-7;-;6-7 +"^(?:a)$" +-;-;-;- +-;-;-;- +"^(?:a)" +-;-;-;- +-;-;-;- +"(?:a)$" +-;-;-;- +-;6-7;-;6-7 +strings +"" +"aa" +regexps +"a+" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:a+)$" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:a+)" +-;-;-;- +0-2;0-2;0-2;0-2 +"(?:a+)$" +-;-;-;- +0-2;0-2;0-2;0-2 +strings +"" +"ab" +regexps +"(a+|b)+" +-;-;-;- +0-2 1-2;0-2 1-2;0-2 1-2;0-2 1-2 +"^(?:(a+|b)+)$" +-;-;-;- +0-2 1-2;0-2 1-2;0-2 1-2;0-2 1-2 +"^(?:(a+|b)+)" +-;-;-;- +0-2 1-2;0-2 1-2;0-2 1-2;0-2 1-2 +"(?:(a+|b)+)$" +-;-;-;- +0-2 1-2;0-2 1-2;0-2 1-2;0-2 1-2 +strings +"" +"xabcdx" +regexps +"ab|cd" +-;-;-;- +-;1-3;-;1-3 +"^(?:ab|cd)$" +-;-;-;- +-;-;-;- +"^(?:ab|cd)" +-;-;-;- +-;-;-;- +"(?:ab|cd)$" +-;-;-;- +-;-;-;- +strings +"" +"hello\ngoodbye\n" +regexps +"h.*od?" +-;-;-;- +-;0-5;-;0-5 +"^(?:h.*od?)$" +-;-;-;- +-;-;-;- +"^(?:h.*od?)" +-;-;-;- +-;0-5;-;0-5 +"(?:h.*od?)$" +-;-;-;- +-;-;-;- +strings +"" +"hello\ngoodbye\n" +regexps +"h.*o" +-;-;-;- +-;0-5;-;0-5 +"^(?:h.*o)$" +-;-;-;- +-;-;-;- +"^(?:h.*o)" +-;-;-;- +-;0-5;-;0-5 +"(?:h.*o)$" +-;-;-;- +-;-;-;- +strings +"" +"goodbye\nhello\n" +regexps +"h.*o" +-;-;-;- +-;8-13;-;8-13 +"^(?:h.*o)$" +-;-;-;- +-;-;-;- +"^(?:h.*o)" +-;-;-;- +-;-;-;- +"(?:h.*o)$" +-;-;-;- +-;-;-;- +strings +"" +"hello world" +regexps +"h.*o" +-;-;-;- +-;0-8;-;0-8 +"^(?:h.*o)$" +-;-;-;- +-;-;-;- +"^(?:h.*o)" +-;-;-;- +-;0-8;-;0-8 +"(?:h.*o)$" +-;-;-;- +-;-;-;- +strings +"" +"othello, world" +regexps +"h.*o" +-;-;-;- +-;2-11;-;2-11 +"^(?:h.*o)$" +-;-;-;- +-;-;-;- +"^(?:h.*o)" +-;-;-;- +-;-;-;- +"(?:h.*o)$" +-;-;-;- +-;-;-;- +strings +"" +"aaaaaaa" +regexps +"[^\\s\\S]" +-;-;-;- +-;-;-;- +"^(?:[^\\s\\S])$" +-;-;-;- +-;-;-;- +"^(?:[^\\s\\S])" +-;-;-;- +-;-;-;- +"(?:[^\\s\\S])$" +-;-;-;- +-;-;-;- +strings +"" +"aaaaaaa" +regexps +"a" +-;-;-;- +-;0-1;-;0-1 +"^(?:a)$" +-;-;-;- +-;-;-;- +"^(?:a)" +-;-;-;- +-;0-1;-;0-1 +"(?:a)$" +-;-;-;- +-;6-7;-;6-7 +strings +"" +"aaaaaaa" +regexps +"a*" +0-0;0-0;0-0;0-0 +0-7;0-7;0-7;0-7 +"^(?:a*)$" +0-0;0-0;0-0;0-0 +0-7;0-7;0-7;0-7 +"^(?:a*)" +0-0;0-0;0-0;0-0 +0-7;0-7;0-7;0-7 +"(?:a*)$" +0-0;0-0;0-0;0-0 +0-7;0-7;0-7;0-7 +strings +"" +"" +regexps +"a*" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:a*)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:a*)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:a*)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"" +regexps +"a*" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:a*)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:a*)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:a*)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"xabcdx" +regexps +"ab|cd" +-;-;-;- +-;1-3;-;1-3 +"^(?:ab|cd)$" +-;-;-;- +-;-;-;- +"^(?:ab|cd)" +-;-;-;- +-;-;-;- +"(?:ab|cd)$" +-;-;-;- +-;-;-;- +strings +"" +"cab" +regexps +"a" +-;-;-;- +-;1-2;-;1-2 +"^(?:a)$" +-;-;-;- +-;-;-;- +"^(?:a)" +-;-;-;- +-;-;-;- +"(?:a)$" +-;-;-;- +-;-;-;- +strings +"" +"cab" +regexps +"a*b" +-;-;-;- +-;1-3;-;1-3 +"^(?:a*b)$" +-;-;-;- +-;-;-;- +"^(?:a*b)" +-;-;-;- +-;-;-;- +"(?:a*b)$" +-;-;-;- +-;1-3;-;1-3 +strings +"" +"x" +regexps +"((((((((((((((((((((x))))))))))))))))))))" +-;-;-;- +0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 +"^(?:((((((((((((((((((((x)))))))))))))))))))))$" +-;-;-;- +0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 +"^(?:((((((((((((((((((((x)))))))))))))))))))))" +-;-;-;- +0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 +"(?:((((((((((((((((((((x)))))))))))))))))))))$" +-;-;-;- +0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1;0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 0-1 +strings +"" +"xxxabcdxxx" +regexps +"[abcd]" +-;-;-;- +-;3-4;-;3-4 +"^(?:[abcd])$" +-;-;-;- +-;-;-;- +"^(?:[abcd])" +-;-;-;- +-;-;-;- +"(?:[abcd])$" +-;-;-;- +-;-;-;- +strings +"" +"xxxabcdxxx" +regexps +"[^x]" +-;-;-;- +-;3-4;-;3-4 +"^(?:[^x])$" +-;-;-;- +-;-;-;- +"^(?:[^x])" +-;-;-;- +-;-;-;- +"(?:[^x])$" +-;-;-;- +-;-;-;- +strings +"" +"xxxabcdxxx" +regexps +"[abcd]+" +-;-;-;- +-;3-7;-;3-7 +"^(?:[abcd]+)$" +-;-;-;- +-;-;-;- +"^(?:[abcd]+)" +-;-;-;- +-;-;-;- +"(?:[abcd]+)$" +-;-;-;- +-;-;-;- +strings +"" +"xxxabcdxxx" +regexps +"[^x]+" +-;-;-;- +-;3-7;-;3-7 +"^(?:[^x]+)$" +-;-;-;- +-;-;-;- +"^(?:[^x]+)" +-;-;-;- +-;-;-;- +"(?:[^x]+)$" +-;-;-;- +-;-;-;- +strings +"" +"fo" +regexps +"(fo|foo)" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:(fo|foo))$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:(fo|foo))" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"(?:(fo|foo))$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +strings +"" +"foo" +regexps +"(foo|fo)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(foo|fo))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(foo|fo))" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:(foo|fo))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"aA" +regexps +"aa" +-;-;-;- +-;-;-;- +"^(?:aa)$" +-;-;-;- +-;-;-;- +"^(?:aa)" +-;-;-;- +-;-;-;- +"(?:aa)$" +-;-;-;- +-;-;-;- +strings +"" +"Aa" +regexps +"a" +-;-;-;- +-;1-2;-;1-2 +"^(?:a)$" +-;-;-;- +-;-;-;- +"^(?:a)" +-;-;-;- +-;-;-;- +"(?:a)$" +-;-;-;- +-;1-2;-;1-2 +strings +"" +"A" +regexps +"a" +-;-;-;- +-;-;-;- +"^(?:a)$" +-;-;-;- +-;-;-;- +"^(?:a)" +-;-;-;- +-;-;-;- +"(?:a)$" +-;-;-;- +-;-;-;- +strings +"" +"abc" +regexps +"ABC" +-;-;-;- +-;-;-;- +"^(?:ABC)$" +-;-;-;- +-;-;-;- +"^(?:ABC)" +-;-;-;- +-;-;-;- +"(?:ABC)$" +-;-;-;- +-;-;-;- +strings +"" +"XABCY" +regexps +"abc" +-;-;-;- +-;-;-;- +"^(?:abc)$" +-;-;-;- +-;-;-;- +"^(?:abc)" +-;-;-;- +-;-;-;- +"(?:abc)$" +-;-;-;- +-;-;-;- +strings +"" +"xabcy" +regexps +"ABC" +-;-;-;- +-;-;-;- +"^(?:ABC)$" +-;-;-;- +-;-;-;- +"^(?:ABC)" +-;-;-;- +-;-;-;- +"(?:ABC)$" +-;-;-;- +-;-;-;- +strings +"" +"foo" +regexps +"foo|bar|[A-Z]" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:foo|bar|[A-Z])$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:foo|bar|[A-Z])" +-;-;-;- +0-3;0-3;0-3;0-3 +"(?:foo|bar|[A-Z])$" +-;-;-;- +0-3;0-3;0-3;0-3 +strings +"" +"foo" +regexps +"^(foo|bar|[A-Z])" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(foo|bar|[A-Z]))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(foo|bar|[A-Z]))" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:^(foo|bar|[A-Z]))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"foo\n" +regexps +"(foo|bar|[A-Z])$" +-;-;-;- +-;-;-;- +"^(?:(foo|bar|[A-Z])$)$" +-;-;-;- +-;-;-;- +"^(?:(foo|bar|[A-Z])$)" +-;-;-;- +-;-;-;- +"(?:(foo|bar|[A-Z])$)$" +-;-;-;- +-;-;-;- +strings +"" +"foo" +regexps +"(foo|bar|[A-Z])$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(foo|bar|[A-Z])$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(foo|bar|[A-Z])$)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:(foo|bar|[A-Z])$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"foo\n" +regexps +"^(foo|bar|[A-Z])$" +-;-;-;- +-;-;-;- +"^(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +-;-;-;- +"^(?:^(foo|bar|[A-Z])$)" +-;-;-;- +-;-;-;- +"(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +-;-;-;- +strings +"" +"foo" +regexps +"^(foo|bar|[A-Z])$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(foo|bar|[A-Z])$)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"bar" +regexps +"^(foo|bar|[A-Z])$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(foo|bar|[A-Z])$)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"X" +regexps +"^(foo|bar|[A-Z])$" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +"^(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +"^(?:^(foo|bar|[A-Z])$)" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +"(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +strings +"" +"XY" +regexps +"^(foo|bar|[A-Z])$" +-;-;-;- +-;-;-;- +"^(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +-;-;-;- +"^(?:^(foo|bar|[A-Z])$)" +-;-;-;- +-;-;-;- +"(?:^(foo|bar|[A-Z])$)$" +-;-;-;- +-;-;-;- +strings +"" +"fo" +regexps +"^(fo|foo)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:^(fo|foo)$)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:^(fo|foo)$)" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"(?:^(fo|foo)$)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +strings +"" +"foo" +regexps +"^(fo|foo)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(fo|foo)$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^(fo|foo)$)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:^(fo|foo)$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"fo" +regexps +"^^(fo|foo)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:^^(fo|foo)$)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:^^(fo|foo)$)" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"(?:^^(fo|foo)$)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +strings +"" +"foo" +regexps +"^^(fo|foo)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^^(fo|foo)$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^^(fo|foo)$)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:^^(fo|foo)$)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"" +regexps +"^$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"" +regexps +"^^$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"" +regexps +"^$$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"^$$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"" +regexps +"^^$$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^$$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^^$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^^$$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^$$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^$$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^^$$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"" +regexps +"^^^^^^^^$$$$$$$$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^^^^^^^$$$$$$$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^^^^^^^$$$$$$$$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^^^^^^^^$$$$$$$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^" +0-0;0-0;0-0;0-0 +-;0-0;-;0-0 +"^(?:^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^)" +0-0;0-0;0-0;0-0 +-;0-0;-;0-0 +"(?:^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"$" +0-0;0-0;0-0;0-0 +-;1-1;-;1-1 +"^(?:$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:$)$" +0-0;0-0;0-0;0-0 +-;1-1;-;1-1 +strings +"" +"nofoo foo that" +regexps +"\\bfoo\\b" +-;-;-;- +-;6-9;-;6-9 +"^(?:\\bfoo\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bfoo\\b)" +-;-;-;- +-;-;-;- +"(?:\\bfoo\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"faoa x" +regexps +"a\\b" +-;-;-;- +-;3-4;-;3-4 +"^(?:a\\b)$" +-;-;-;- +-;-;-;- +"^(?:a\\b)" +-;-;-;- +-;-;-;- +"(?:a\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"bar x" +regexps +"\\bbar" +-;-;-;- +-;0-3;-;0-3 +"^(?:\\bbar)$" +-;-;-;- +-;-;-;- +"^(?:\\bbar)" +-;-;-;- +-;0-3;-;0-3 +"(?:\\bbar)$" +-;-;-;- +-;-;-;- +strings +"" +"foo\nbar x" +regexps +"\\bbar" +-;-;-;- +-;4-7;-;4-7 +"^(?:\\bbar)$" +-;-;-;- +-;-;-;- +"^(?:\\bbar)" +-;-;-;- +-;-;-;- +"(?:\\bbar)$" +-;-;-;- +-;-;-;- +strings +"" +"foobar" +regexps +"bar\\b" +-;-;-;- +-;3-6;-;3-6 +"^(?:bar\\b)$" +-;-;-;- +-;-;-;- +"^(?:bar\\b)" +-;-;-;- +-;-;-;- +"(?:bar\\b)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"foobar\nxxx" +regexps +"bar\\b" +-;-;-;- +-;3-6;-;3-6 +"^(?:bar\\b)$" +-;-;-;- +-;-;-;- +"^(?:bar\\b)" +-;-;-;- +-;-;-;- +"(?:bar\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"foo" +regexps +"(foo|bar|[A-Z])\\b" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(foo|bar|[A-Z])\\b)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"foo\n" +regexps +"(foo|bar|[A-Z])\\b" +-;-;-;- +-;0-3 0-3;-;0-3 0-3 +"^(?:(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +"^(?:(foo|bar|[A-Z])\\b)" +-;-;-;- +-;0-3 0-3;-;0-3 0-3 +"(?:(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"\\b" +-;-;-;- +-;-;-;- +"^(?:\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b)" +-;-;-;- +-;-;-;- +"(?:\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"\\b" +-;-;-;- +-;0-0;-;0-0 +"^(?:\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b)" +-;-;-;- +-;0-0;-;0-0 +"(?:\\b)$" +-;-;-;- +-;1-1;-;1-1 +strings +"" +"foo" +regexps +"\\b(foo|bar|[A-Z])" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z]))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z]))" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:\\b(foo|bar|[A-Z]))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"X" +regexps +"\\b(foo|bar|[A-Z])\\b" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +"^(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +"^(?:\\b(foo|bar|[A-Z])\\b)" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +"(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-1 0-1;0-1 0-1;0-1 0-1;0-1 0-1 +strings +"" +"XY" +regexps +"\\b(foo|bar|[A-Z])\\b" +-;-;-;- +-;-;-;- +"^(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b(foo|bar|[A-Z])\\b)" +-;-;-;- +-;-;-;- +"(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"bar" +regexps +"\\b(foo|bar|[A-Z])\\b" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z])\\b)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"foo" +regexps +"\\b(foo|bar|[A-Z])\\b" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z])\\b)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"foo\n" +regexps +"\\b(foo|bar|[A-Z])\\b" +-;-;-;- +-;0-3 0-3;-;0-3 0-3 +"^(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b(foo|bar|[A-Z])\\b)" +-;-;-;- +-;0-3 0-3;-;0-3 0-3 +"(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"ffoo bbar N x" +regexps +"\\b(foo|bar|[A-Z])\\b" +-;-;-;- +-;10-11 10-11;-;10-11 10-11 +"^(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b(foo|bar|[A-Z])\\b)" +-;-;-;- +-;-;-;- +"(?:\\b(foo|bar|[A-Z])\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"fo" +regexps +"\\b(fo|foo)\\b" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:\\b(fo|foo)\\b)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:\\b(fo|foo)\\b)" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"(?:\\b(fo|foo)\\b)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +strings +"" +"foo" +regexps +"\\b(fo|foo)\\b" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(fo|foo)\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:\\b(fo|foo)\\b)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:\\b(fo|foo)\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"" +regexps +"\\b\\b" +-;-;-;- +-;-;-;- +"^(?:\\b\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b\\b)" +-;-;-;- +-;-;-;- +"(?:\\b\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"\\b\\b" +-;-;-;- +-;0-0;-;0-0 +"^(?:\\b\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\b\\b)" +-;-;-;- +-;0-0;-;0-0 +"(?:\\b\\b)$" +-;-;-;- +-;1-1;-;1-1 +strings +"" +"" +regexps +"\\b$" +-;-;-;- +-;-;-;- +"^(?:\\b$)$" +-;-;-;- +-;-;-;- +"^(?:\\b$)" +-;-;-;- +-;-;-;- +"(?:\\b$)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"\\b$" +-;-;-;- +-;1-1;-;1-1 +"^(?:\\b$)$" +-;-;-;- +-;-;-;- +"^(?:\\b$)" +-;-;-;- +-;-;-;- +"(?:\\b$)$" +-;-;-;- +-;1-1;-;1-1 +strings +"" +"y x" +regexps +"\\b$" +-;-;-;- +-;3-3;-;3-3 +"^(?:\\b$)$" +-;-;-;- +-;-;-;- +"^(?:\\b$)" +-;-;-;- +-;-;-;- +"(?:\\b$)$" +-;-;-;- +-;3-3;-;3-3 +strings +"" +"x" +regexps +"\\b.$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\b.$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\b.$)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\b.$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"fo" +regexps +"^\\b(fo|foo)\\b" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:^\\b(fo|foo)\\b)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"^(?:^\\b(fo|foo)\\b)" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +"(?:^\\b(fo|foo)\\b)$" +-;-;-;- +0-2 0-2;0-2 0-2;0-2 0-2;0-2 0-2 +strings +"" +"foo" +regexps +"^\\b(fo|foo)\\b" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^\\b(fo|foo)\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:^\\b(fo|foo)\\b)" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"(?:^\\b(fo|foo)\\b)$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"" +regexps +"^\\b" +-;-;-;- +-;-;-;- +"^(?:^\\b)$" +-;-;-;- +-;-;-;- +"^(?:^\\b)" +-;-;-;- +-;-;-;- +"(?:^\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^\\b" +-;-;-;- +-;0-0;-;0-0 +"^(?:^\\b)$" +-;-;-;- +-;-;-;- +"^(?:^\\b)" +-;-;-;- +-;0-0;-;0-0 +"(?:^\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"^\\b\\b" +-;-;-;- +-;-;-;- +"^(?:^\\b\\b)$" +-;-;-;- +-;-;-;- +"^(?:^\\b\\b)" +-;-;-;- +-;-;-;- +"(?:^\\b\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^\\b\\b" +-;-;-;- +-;0-0;-;0-0 +"^(?:^\\b\\b)$" +-;-;-;- +-;-;-;- +"^(?:^\\b\\b)" +-;-;-;- +-;0-0;-;0-0 +"(?:^\\b\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"^\\b$" +-;-;-;- +-;-;-;- +"^(?:^\\b$)$" +-;-;-;- +-;-;-;- +"^(?:^\\b$)" +-;-;-;- +-;-;-;- +"(?:^\\b$)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^\\b$" +-;-;-;- +-;-;-;- +"^(?:^\\b$)$" +-;-;-;- +-;-;-;- +"^(?:^\\b$)" +-;-;-;- +-;-;-;- +"(?:^\\b$)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^\\b.$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:^\\b.$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:^\\b.$)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:^\\b.$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"x" +regexps +"^\\b.\\b$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:^\\b.\\b$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:^\\b.\\b$)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:^\\b.\\b$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"" +regexps +"^^^^^^^^\\b$$$$$$$" +-;-;-;- +-;-;-;- +"^(?:^^^^^^^^\\b$$$$$$$)$" +-;-;-;- +-;-;-;- +"^(?:^^^^^^^^\\b$$$$$$$)" +-;-;-;- +-;-;-;- +"(?:^^^^^^^^\\b$$$$$$$)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^^^^^^^^\\b.$$$$$$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:^^^^^^^^\\b.$$$$$$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:^^^^^^^^\\b.$$$$$$)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:^^^^^^^^\\b.$$$$$$)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"x" +regexps +"^^^^^^^^\\b$$$$$$$" +-;-;-;- +-;-;-;- +"^(?:^^^^^^^^\\b$$$$$$$)$" +-;-;-;- +-;-;-;- +"^(?:^^^^^^^^\\b$$$$$$$)" +-;-;-;- +-;-;-;- +"(?:^^^^^^^^\\b$$$$$$$)$" +-;-;-;- +-;-;-;- +strings +"" +"n foo xfoox that" +regexps +"\\Bfoo\\B" +-;-;-;- +-;7-10;-;7-10 +"^(?:\\Bfoo\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\Bfoo\\B)" +-;-;-;- +-;-;-;- +"(?:\\Bfoo\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"faoa x" +regexps +"a\\B" +-;-;-;- +-;1-2;-;1-2 +"^(?:a\\B)$" +-;-;-;- +-;-;-;- +"^(?:a\\B)" +-;-;-;- +-;-;-;- +"(?:a\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"bar x" +regexps +"\\Bbar" +-;-;-;- +-;-;-;- +"^(?:\\Bbar)$" +-;-;-;- +-;-;-;- +"^(?:\\Bbar)" +-;-;-;- +-;-;-;- +"(?:\\Bbar)$" +-;-;-;- +-;-;-;- +strings +"" +"foo\nbar x" +regexps +"\\Bbar" +-;-;-;- +-;-;-;- +"^(?:\\Bbar)$" +-;-;-;- +-;-;-;- +"^(?:\\Bbar)" +-;-;-;- +-;-;-;- +"(?:\\Bbar)$" +-;-;-;- +-;-;-;- +strings +"" +"foobar" +regexps +"bar\\B" +-;-;-;- +-;-;-;- +"^(?:bar\\B)$" +-;-;-;- +-;-;-;- +"^(?:bar\\B)" +-;-;-;- +-;-;-;- +"(?:bar\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"foobar\nxxx" +regexps +"bar\\B" +-;-;-;- +-;-;-;- +"^(?:bar\\B)$" +-;-;-;- +-;-;-;- +"^(?:bar\\B)" +-;-;-;- +-;-;-;- +"(?:bar\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"foox" +regexps +"(foo|bar|[A-Z])\\B" +-;-;-;- +-;0-3 0-3;-;0-3 0-3 +"^(?:(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:(foo|bar|[A-Z])\\B)" +-;-;-;- +-;0-3 0-3;-;0-3 0-3 +"(?:(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"foo\n" +regexps +"(foo|bar|[A-Z])\\B" +-;-;-;- +-;-;-;- +"^(?:(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"\\B" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:\\B)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"\\B" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"foo" +regexps +"\\B(foo|bar|[A-Z])" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z]))$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z]))" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z]))$" +-;-;-;- +-;-;-;- +strings +"" +"xXy" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;1-2 1-2;-;1-2 1-2 +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"XY" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"XYZ" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;1-2 1-2;-;1-2 1-2 +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"abara" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;1-4 1-4;-;1-4 1-4 +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"xfoo_" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;1-4 1-4;-;1-4 1-4 +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"xfoo\n" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"foo bar vNx" +regexps +"\\B(foo|bar|[A-Z])\\B" +-;-;-;- +-;9-10 9-10;-;9-10 9-10 +"^(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|bar|[A-Z])\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|bar|[A-Z])\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"xfoo" +regexps +"\\B(fo|foo)\\B" +-;-;-;- +-;1-3 1-3;-;1-3 1-3 +"^(?:\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(fo|foo)\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"xfooo" +regexps +"\\B(foo|fo)\\B" +-;-;-;- +-;1-4 1-4;-;1-4 1-4 +"^(?:\\B(foo|fo)\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(foo|fo)\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(foo|fo)\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"\\B\\B" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:\\B\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:\\B\\B)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:\\B\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"\\B\\B" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B\\B)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:\\B\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"" +regexps +"\\B$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:\\B$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:\\B$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:\\B$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"\\B$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:\\B$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"y x" +regexps +"\\B$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:\\B$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:\\B$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"\\B.$" +-;-;-;- +-;-;-;- +"^(?:\\B.$)$" +-;-;-;- +-;-;-;- +"^(?:\\B.$)" +-;-;-;- +-;-;-;- +"(?:\\B.$)$" +-;-;-;- +-;-;-;- +strings +"" +"fo" +regexps +"^\\B(fo|foo)\\B" +-;-;-;- +-;-;-;- +"^(?:^\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +"^(?:^\\B(fo|foo)\\B)" +-;-;-;- +-;-;-;- +"(?:^\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"foo" +regexps +"^\\B(fo|foo)\\B" +-;-;-;- +-;-;-;- +"^(?:^\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +"^(?:^\\B(fo|foo)\\B)" +-;-;-;- +-;-;-;- +"(?:^\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"^\\B" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^\\B)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^\\B" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^\\B)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"" +regexps +"^\\B\\B" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^\\B\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^\\B\\B)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^\\B\\B)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^\\B\\B" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^\\B\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^\\B\\B)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^\\B\\B)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"" +regexps +"^\\B$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^\\B$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^\\B$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^\\B$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^\\B$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^\\B$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^\\B$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^\\B$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"^\\B.$" +-;-;-;- +-;-;-;- +"^(?:^\\B.$)$" +-;-;-;- +-;-;-;- +"^(?:^\\B.$)" +-;-;-;- +-;-;-;- +"(?:^\\B.$)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^\\B.\\B$" +-;-;-;- +-;-;-;- +"^(?:^\\B.\\B$)$" +-;-;-;- +-;-;-;- +"^(?:^\\B.\\B$)" +-;-;-;- +-;-;-;- +"(?:^\\B.\\B$)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"^^^^^^^^\\B$$$$$$$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^^^^^^^\\B$$$$$$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^^^^^^^^\\B$$$$$$$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^^^^^^^^\\B$$$$$$$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^^^^^^^^\\B.$$$$$$" +-;-;-;- +-;-;-;- +"^(?:^^^^^^^^\\B.$$$$$$)$" +-;-;-;- +-;-;-;- +"^(?:^^^^^^^^\\B.$$$$$$)" +-;-;-;- +-;-;-;- +"(?:^^^^^^^^\\B.$$$$$$)$" +-;-;-;- +-;-;-;- +strings +"" +"x" +regexps +"^^^^^^^^\\B$$$$$$$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^^^^^^^\\B$$$$$$$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^^^^^^^\\B$$$$$$$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^^^^^^^^\\B$$$$$$$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"\\bx\\b" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\bx\\b)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\bx\\b)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\bx\\b)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"x>" +regexps +"\\bx\\b" +-;-;-;- +-;0-1;-;0-1 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;0-1;-;0-1 +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"<x" +regexps +"\\bx\\b" +-;-;-;- +-;1-2;-;1-2 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;1-2;-;1-2 +strings +"" +"<x>" +regexps +"\\bx\\b" +-;-;-;- +-;1-2;-;1-2 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"ax" +regexps +"\\bx\\b" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"xb" +regexps +"\\bx\\b" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"axb" +regexps +"\\bx\\b" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"«x" +regexps +"\\bx\\b" +-;-;-;- +-;2-3;-;2-3 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;2-3;-;2-3 +strings +"" +"x»" +regexps +"\\bx\\b" +-;-;-;- +-;0-1;-;0-1 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;0-1;-;0-1 +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"«x»" +regexps +"\\bx\\b" +-;-;-;- +-;2-3;-;2-3 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"axb" +regexps +"\\bx\\b" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"áxβ" +regexps +"\\bx\\b" +-;-;-;- +-;2-3;-;2-3 +"^(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +"^(?:\\bx\\b)" +-;-;-;- +-;-;-;- +"(?:\\bx\\b)$" +-;-;-;- +-;-;-;- +strings +"" +"axb" +regexps +"\\Bx\\B" +-;-;-;- +-;1-2;-;1-2 +"^(?:\\Bx\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\Bx\\B)" +-;-;-;- +-;-;-;- +"(?:\\Bx\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"áxβ" +regexps +"\\Bx\\B" +-;-;-;- +-;-;-;- +"^(?:\\Bx\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\Bx\\B)" +-;-;-;- +-;-;-;- +"(?:\\Bx\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"" +regexps +"^$^$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$^$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^$^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"" +regexps +"^$^" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$^)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:^$^)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:^$^)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"" +regexps +"$^$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:$^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"^(?:$^$)" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +"(?:$^$)$" +0-0;0-0;0-0;0-0 +0-0;0-0;0-0;0-0 +strings +"" +"x" +regexps +"^$^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"^$^" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x" +regexps +"$^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x\ny" +regexps +"^$^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x\ny" +regexps +"^$^" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x\ny" +regexps +"$^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x\n\ny" +regexps +"^$^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x\n\ny" +regexps +"^$^" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^$^)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:^$^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"x\n\ny" +regexps +"$^$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:$^$)" +0-0;0-0;0-0;0-0 +-;-;-;- +"(?:$^$)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"foo$bar" +regexps +"^(foo\\$)$" +-;-;-;- +-;-;-;- +"^(?:^(foo\\$)$)$" +-;-;-;- +-;-;-;- +"^(?:^(foo\\$)$)" +-;-;-;- +-;-;-;- +"(?:^(foo\\$)$)$" +-;-;-;- +-;-;-;- +strings +"" +"foo$bar" +regexps +"(foo\\$)" +-;-;-;- +-;0-4 0-4;-;0-4 0-4 +"^(?:(foo\\$))$" +-;-;-;- +-;-;-;- +"^(?:(foo\\$))" +-;-;-;- +-;0-4 0-4;-;0-4 0-4 +"(?:(foo\\$))$" +-;-;-;- +-;-;-;- +strings +"" +"abc" +regexps +"^...$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:^...$)$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:^...$)" +-;-;-;- +0-3;0-3;0-3;0-3 +"(?:^...$)$" +-;-;-;- +0-3;0-3;0-3;0-3 +strings +"" +"本" +regexps +"^本$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:^本$)$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:^本$)" +-;-;-;- +0-3;0-3;0-3;0-3 +"(?:^本$)$" +-;-;-;- +0-3;0-3;0-3;0-3 +strings +"" +"日本語" +regexps +"^...$" +-;-;-;- +0-9;0-9;0-9;0-9 +"^(?:^...$)$" +-;-;-;- +0-9;0-9;0-9;0-9 +"^(?:^...$)" +-;-;-;- +0-9;0-9;0-9;0-9 +"(?:^...$)$" +-;-;-;- +0-9;0-9;0-9;0-9 +strings +"" +".本." +regexps +"^...$" +-;-;-;- +0-5;0-5;0-5;0-5 +"^(?:^...$)$" +-;-;-;- +0-5;0-5;0-5;0-5 +"^(?:^...$)" +-;-;-;- +0-5;0-5;0-5;0-5 +"(?:^...$)$" +-;-;-;- +0-5;0-5;0-5;0-5 +strings +"" +"本" +regexps +"^\\C\\C\\C$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:^\\C\\C\\C$)$" +-;-;-;- +0-3;0-3;0-3;0-3 +"^(?:^\\C\\C\\C$)" +-;-;-;- +0-3;0-3;0-3;0-3 +"(?:^\\C\\C\\C$)$" +-;-;-;- +0-3;0-3;0-3;0-3 +strings +"" +"本" +regexps +"^\\C$" +-;-;-;- +-;-;-;- +"^(?:^\\C$)$" +-;-;-;- +-;-;-;- +"^(?:^\\C$)" +-;-;-;- +-;-;-;- +"(?:^\\C$)$" +-;-;-;- +-;-;-;- +strings +"" +"日本語" +regexps +"^\\C\\C\\C$" +-;-;-;- +-;-;-;- +"^(?:^\\C\\C\\C$)$" +-;-;-;- +-;-;-;- +"^(?:^\\C\\C\\C$)" +-;-;-;- +-;-;-;- +"(?:^\\C\\C\\C$)$" +-;-;-;- +-;-;-;- +strings +"" +"日本語" +regexps +"^...$" +-;-;-;- +0-9;0-9;0-9;0-9 +"^(?:^...$)$" +-;-;-;- +0-9;0-9;0-9;0-9 +"^(?:^...$)" +-;-;-;- +0-9;0-9;0-9;0-9 +"(?:^...$)$" +-;-;-;- +0-9;0-9;0-9;0-9 +strings +"" +"日本語" +regexps +"^.........$" +-;-;-;- +-;-;-;- +"^(?:^.........$)$" +-;-;-;- +-;-;-;- +"^(?:^.........$)" +-;-;-;- +-;-;-;- +"(?:^.........$)$" +-;-;-;- +-;-;-;- +strings +"" +".本." +regexps +"^...$" +-;-;-;- +0-5;0-5;0-5;0-5 +"^(?:^...$)$" +-;-;-;- +0-5;0-5;0-5;0-5 +"^(?:^...$)" +-;-;-;- +0-5;0-5;0-5;0-5 +"(?:^...$)$" +-;-;-;- +0-5;0-5;0-5;0-5 +strings +"" +".本." +regexps +"^.....$" +-;-;-;- +-;-;-;- +"^(?:^.....$)$" +-;-;-;- +-;-;-;- +"^(?:^.....$)" +-;-;-;- +-;-;-;- +"(?:^.....$)$" +-;-;-;- +-;-;-;- +strings +"" +"xfooo" +regexps +"\\B(fo|foo)\\B" +-;-;-;- +-;1-3 1-3;-;1-4 1-4 +"^(?:\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +"^(?:\\B(fo|foo)\\B)" +-;-;-;- +-;-;-;- +"(?:\\B(fo|foo)\\B)$" +-;-;-;- +-;-;-;- +strings +"" +"foo" +regexps +"(fo|foo)" +-;-;-;- +0-3 0-3;0-2 0-2;0-3 0-3;0-3 0-3 +"^(?:(fo|foo))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +"^(?:(fo|foo))" +-;-;-;- +0-3 0-3;0-2 0-2;0-3 0-3;0-3 0-3 +"(?:(fo|foo))$" +-;-;-;- +0-3 0-3;0-3 0-3;0-3 0-3;0-3 0-3 +strings +"" +"a" +regexps +"\\141" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\141)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\141)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\141)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"0" +regexps +"\\060" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\060)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\060)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\060)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"00" +regexps +"\\0600" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:\\0600)$" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:\\0600)" +-;-;-;- +0-2;0-2;0-2;0-2 +"(?:\\0600)$" +-;-;-;- +0-2;0-2;0-2;0-2 +strings +"" +"08" +regexps +"\\608" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:\\608)$" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:\\608)" +-;-;-;- +0-2;0-2;0-2;0-2 +"(?:\\608)$" +-;-;-;- +0-2;0-2;0-2;0-2 +strings +"" +"" +regexps +"\\01" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\01)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\01)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\01)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"8" +regexps +"\\018" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:\\018)$" +-;-;-;- +0-2;0-2;0-2;0-2 +"^(?:\\018)" +-;-;-;- +0-2;0-2;0-2;0-2 +"(?:\\018)$" +-;-;-;- +0-2;0-2;0-2;0-2 +strings +"" +"a" +regexps +"\\x{61}" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\x{61})$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\x{61})" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\x{61})$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"a" +regexps +"\\x61" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\x61)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\x61)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\x61)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"a" +regexps +"\\x{00000061}" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\x{00000061})$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:\\x{00000061})" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:\\x{00000061})$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"aαβb" +regexps +"\\p{Greek}+" +-;-;-;- +-;1-5;-;1-5 +"^(?:\\p{Greek}+)$" +-;-;-;- +-;-;-;- +"^(?:\\p{Greek}+)" +-;-;-;- +-;-;-;- +"(?:\\p{Greek}+)$" +-;-;-;- +-;-;-;- +strings +"" +"aαβb" +regexps +"\\P{Greek}+" +-;-;-;- +-;0-1;-;0-1 +"^(?:\\P{Greek}+)$" +-;-;-;- +-;-;-;- +"^(?:\\P{Greek}+)" +-;-;-;- +-;0-1;-;0-1 +"(?:\\P{Greek}+)$" +-;-;-;- +-;5-6;-;5-6 +strings +"" +"aαβb" +regexps +"\\p{^Greek}+" +-;-;-;- +-;0-1;-;0-1 +"^(?:\\p{^Greek}+)$" +-;-;-;- +-;-;-;- +"^(?:\\p{^Greek}+)" +-;-;-;- +-;0-1;-;0-1 +"(?:\\p{^Greek}+)$" +-;-;-;- +-;5-6;-;5-6 +strings +"" +"aαβb" +regexps +"\\P{^Greek}+" +-;-;-;- +-;1-5;-;1-5 +"^(?:\\P{^Greek}+)$" +-;-;-;- +-;-;-;- +"^(?:\\P{^Greek}+)" +-;-;-;- +-;-;-;- +"(?:\\P{^Greek}+)$" +-;-;-;- +-;-;-;- +strings +"" +"abc123" +regexps +"[^0-9]+" +-;-;-;- +-;0-3;-;0-3 +"^(?:[^0-9]+)$" +-;-;-;- +-;-;-;- +"^(?:[^0-9]+)" +-;-;-;- +-;0-3;-;0-3 +"(?:[^0-9]+)$" +-;-;-;- +-;-;-;- +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\p{Nd}+" +-;-;-;- +-;3-6;-;3-6 +"^(?:\\p{Nd}+)$" +-;-;-;- +-;-;-;- +"^(?:\\p{Nd}+)" +-;-;-;- +-;-;-;- +"(?:\\p{Nd}+)$" +-;-;-;- +-;-;-;- +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\p{^Nd}+" +-;-;-;- +-;0-3;-;0-3 +"^(?:\\p{^Nd}+)$" +-;-;-;- +-;-;-;- +"^(?:\\p{^Nd}+)" +-;-;-;- +-;0-3;-;0-3 +"(?:\\p{^Nd}+)$" +-;-;-;- +-;6-22;-;6-22 +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\P{Nd}+" +-;-;-;- +-;0-3;-;0-3 +"^(?:\\P{Nd}+)$" +-;-;-;- +-;-;-;- +"^(?:\\P{Nd}+)" +-;-;-;- +-;0-3;-;0-3 +"(?:\\P{Nd}+)$" +-;-;-;- +-;6-22;-;6-22 +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\P{^Nd}+" +-;-;-;- +-;3-6;-;3-6 +"^(?:\\P{^Nd}+)$" +-;-;-;- +-;-;-;- +"^(?:\\P{^Nd}+)" +-;-;-;- +-;-;-;- +"(?:\\P{^Nd}+)$" +-;-;-;- +-;-;-;- +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\pN+" +-;-;-;- +-;3-22;-;3-22 +"^(?:\\pN+)$" +-;-;-;- +-;-;-;- +"^(?:\\pN+)" +-;-;-;- +-;-;-;- +"(?:\\pN+)$" +-;-;-;- +-;3-22;-;3-22 +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\p{N}+" +-;-;-;- +-;3-22;-;3-22 +"^(?:\\p{N}+)$" +-;-;-;- +-;-;-;- +"^(?:\\p{N}+)" +-;-;-;- +-;-;-;- +"(?:\\p{N}+)$" +-;-;-;- +-;3-22;-;3-22 +strings +"" +"abc123²³¼½¾₀₉" +regexps +"\\p{^N}+" +-;-;-;- +-;0-3;-;0-3 +"^(?:\\p{^N}+)$" +-;-;-;- +-;-;-;- +"^(?:\\p{^N}+)" +-;-;-;- +-;0-3;-;0-3 +"(?:\\p{^N}+)$" +-;-;-;- +-;-;-;- +strings +"" +"abc123" +regexps +"\\p{Any}+" +-;-;-;- +0-6;0-6;0-6;0-6 +"^(?:\\p{Any}+)$" +-;-;-;- +0-6;0-6;0-6;0-6 +"^(?:\\p{Any}+)" +-;-;-;- +0-6;0-6;0-6;0-6 +"(?:\\p{Any}+)$" +-;-;-;- +0-6;0-6;0-6;0-6 +strings +"" +"@AaB" +regexps +"(?i)[@-A]+" +-;-;-;- +-;0-3;-;0-3 +"^(?:(?i)[@-A]+)$" +-;-;-;- +-;-;-;- +"^(?:(?i)[@-A]+)" +-;-;-;- +-;0-3;-;0-3 +"(?:(?i)[@-A]+)$" +-;-;-;- +-;-;-;- +strings +"" +"aAzZ" +regexps +"(?i)[A-Z]+" +-;-;-;- +0-4;0-4;0-4;0-4 +"^(?:(?i)[A-Z]+)$" +-;-;-;- +0-4;0-4;0-4;0-4 +"^(?:(?i)[A-Z]+)" +-;-;-;- +0-4;0-4;0-4;0-4 +"(?:(?i)[A-Z]+)$" +-;-;-;- +0-4;0-4;0-4;0-4 +strings +"" +"Aa\\" +regexps +"(?i)[^\\\\]+" +-;-;-;- +-;0-2;-;0-2 +"^(?:(?i)[^\\\\]+)$" +-;-;-;- +-;-;-;- +"^(?:(?i)[^\\\\]+)" +-;-;-;- +-;0-2;-;0-2 +"(?:(?i)[^\\\\]+)$" +-;-;-;- +-;-;-;- +strings +"" +"acegikmoqsuwyACEGIKMOQSUWY" +regexps +"(?i)[acegikmoqsuwy]+" +-;-;-;- +0-26;0-26;0-26;0-26 +"^(?:(?i)[acegikmoqsuwy]+)$" +-;-;-;- +0-26;0-26;0-26;0-26 +"^(?:(?i)[acegikmoqsuwy]+)" +-;-;-;- +0-26;0-26;0-26;0-26 +"(?:(?i)[acegikmoqsuwy]+)$" +-;-;-;- +0-26;0-26;0-26;0-26 +strings +"" +"@AaB" +regexps +"[@-A]+" +-;-;-;- +-;0-2;-;0-2 +"^(?:[@-A]+)$" +-;-;-;- +-;-;-;- +"^(?:[@-A]+)" +-;-;-;- +-;0-2;-;0-2 +"(?:[@-A]+)$" +-;-;-;- +-;-;-;- +strings +"" +"aAzZ" +regexps +"[A-Z]+" +-;-;-;- +-;1-2;-;1-2 +"^(?:[A-Z]+)$" +-;-;-;- +-;-;-;- +"^(?:[A-Z]+)" +-;-;-;- +-;-;-;- +"(?:[A-Z]+)$" +-;-;-;- +-;3-4;-;3-4 +strings +"" +"Aa\\" +regexps +"[^\\\\]+" +-;-;-;- +-;0-2;-;0-2 +"^(?:[^\\\\]+)$" +-;-;-;- +-;-;-;- +"^(?:[^\\\\]+)" +-;-;-;- +-;0-2;-;0-2 +"(?:[^\\\\]+)$" +-;-;-;- +-;-;-;- +strings +"" +"acegikmoqsuwyACEGIKMOQSUWY" +regexps +"[acegikmoqsuwy]+" +-;-;-;- +-;0-13;-;0-13 +"^(?:[acegikmoqsuwy]+)$" +-;-;-;- +-;-;-;- +"^(?:[acegikmoqsuwy]+)" +-;-;-;- +-;0-13;-;0-13 +"(?:[acegikmoqsuwy]+)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"^abc" +-;-;-;- +-;0-3;-;0-3 +"^(?:^abc)$" +-;-;-;- +-;-;-;- +"^(?:^abc)" +-;-;-;- +-;0-3;-;0-3 +"(?:^abc)$" +-;-;-;- +-;-;-;- +strings +"" +"aabcdef" +regexps +"^abc" +-;-;-;- +-;-;-;- +"^(?:^abc)$" +-;-;-;- +-;-;-;- +"^(?:^abc)" +-;-;-;- +-;-;-;- +"(?:^abc)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"^[ay]*[bx]+c" +-;-;-;- +-;0-3;-;0-3 +"^(?:^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +"^(?:^[ay]*[bx]+c)" +-;-;-;- +-;0-3;-;0-3 +"(?:^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +strings +"" +"aabcdef" +regexps +"^[ay]*[bx]+c" +-;-;-;- +-;0-4;-;0-4 +"^(?:^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +"^(?:^[ay]*[bx]+c)" +-;-;-;- +-;0-4;-;0-4 +"(?:^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"def$" +-;-;-;- +-;3-6;-;3-6 +"^(?:def$)$" +-;-;-;- +-;-;-;- +"^(?:def$)" +-;-;-;- +-;-;-;- +"(?:def$)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"abcdeff" +regexps +"def$" +-;-;-;- +-;-;-;- +"^(?:def$)$" +-;-;-;- +-;-;-;- +"^(?:def$)" +-;-;-;- +-;-;-;- +"(?:def$)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"d[ex][fy]$" +-;-;-;- +-;3-6;-;3-6 +"^(?:d[ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:d[ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:d[ex][fy]$)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"abcdeff" +regexps +"d[ex][fy]$" +-;-;-;- +-;-;-;- +"^(?:d[ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:d[ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:d[ex][fy]$)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"[dz][ex][fy]$" +-;-;-;- +-;3-6;-;3-6 +"^(?:[dz][ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:[dz][ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:[dz][ex][fy]$)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"abcdeff" +regexps +"[dz][ex][fy]$" +-;-;-;- +-;-;-;- +"^(?:[dz][ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:[dz][ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:[dz][ex][fy]$)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"(?m)^abc" +-;-;-;- +-;0-3;-;0-3 +"^(?:(?m)^abc)$" +-;-;-;- +-;-;-;- +"^(?:(?m)^abc)" +-;-;-;- +-;0-3;-;0-3 +"(?:(?m)^abc)$" +-;-;-;- +-;-;-;- +strings +"" +"aabcdef" +regexps +"(?m)^abc" +-;-;-;- +-;-;-;- +"^(?:(?m)^abc)$" +-;-;-;- +-;-;-;- +"^(?:(?m)^abc)" +-;-;-;- +-;-;-;- +"(?:(?m)^abc)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"(?m)^[ay]*[bx]+c" +-;-;-;- +-;0-3;-;0-3 +"^(?:(?m)^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +"^(?:(?m)^[ay]*[bx]+c)" +-;-;-;- +-;0-3;-;0-3 +"(?:(?m)^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +strings +"" +"aabcdef" +regexps +"(?m)^[ay]*[bx]+c" +-;-;-;- +-;0-4;-;0-4 +"^(?:(?m)^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +"^(?:(?m)^[ay]*[bx]+c)" +-;-;-;- +-;0-4;-;0-4 +"(?:(?m)^[ay]*[bx]+c)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"(?m)def$" +-;-;-;- +-;3-6;-;3-6 +"^(?:(?m)def$)$" +-;-;-;- +-;-;-;- +"^(?:(?m)def$)" +-;-;-;- +-;-;-;- +"(?:(?m)def$)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"abcdeff" +regexps +"(?m)def$" +-;-;-;- +-;-;-;- +"^(?:(?m)def$)$" +-;-;-;- +-;-;-;- +"^(?:(?m)def$)" +-;-;-;- +-;-;-;- +"(?:(?m)def$)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"(?m)d[ex][fy]$" +-;-;-;- +-;3-6;-;3-6 +"^(?:(?m)d[ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:(?m)d[ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:(?m)d[ex][fy]$)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"abcdeff" +regexps +"(?m)d[ex][fy]$" +-;-;-;- +-;-;-;- +"^(?:(?m)d[ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:(?m)d[ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:(?m)d[ex][fy]$)$" +-;-;-;- +-;-;-;- +strings +"" +"abcdef" +regexps +"(?m)[dz][ex][fy]$" +-;-;-;- +-;3-6;-;3-6 +"^(?:(?m)[dz][ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:(?m)[dz][ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:(?m)[dz][ex][fy]$)$" +-;-;-;- +-;3-6;-;3-6 +strings +"" +"abcdeff" +regexps +"(?m)[dz][ex][fy]$" +-;-;-;- +-;-;-;- +"^(?:(?m)[dz][ex][fy]$)$" +-;-;-;- +-;-;-;- +"^(?:(?m)[dz][ex][fy]$)" +-;-;-;- +-;-;-;- +"(?:(?m)[dz][ex][fy]$)$" +-;-;-;- +-;-;-;- +strings +"" +"a" +regexps +"^" +0-0;0-0;0-0;0-0 +-;0-0;-;0-0 +"^(?:^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^)" +0-0;0-0;0-0;0-0 +-;0-0;-;0-0 +"(?:^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"a" +regexps +"^^" +0-0;0-0;0-0;0-0 +-;0-0;-;0-0 +"^(?:^^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +"^(?:^^)" +0-0;0-0;0-0;0-0 +-;0-0;-;0-0 +"(?:^^)$" +0-0;0-0;0-0;0-0 +-;-;-;- +strings +"" +"a" +regexps +"a" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:a)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:a)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:a)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"a" +regexps +"ab*" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:ab*)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:ab*)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:ab*)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"a" +regexps +"a\\C*" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:a\\C*)$" +-;-;-;- +0-1;0-1;0-1;0-1 +"^(?:a\\C*)" +-;-;-;- +0-1;0-1;0-1;0-1 +"(?:a\\C*)$" +-;-;-;- +0-1;0-1;0-1;0-1 +strings +"" +"baba" +regexps +"a\\C*|ba\\C" +-;-;-;- +-;0-3;-;0-3 +"^(?:a\\C*|ba\\C)$" +-;-;-;- +-;-;-;- +"^(?:a\\C*|ba\\C)" +-;-;-;- +-;0-3;-;0-3 +"(?:a\\C*|ba\\C)$" +-;-;-;- +-;1-4;-;1-4 diff --git a/libgo/go/regexp/testdata/repetition.dat b/libgo/go/regexp/testdata/repetition.dat new file mode 100644 index 0000000..e6361f5 --- /dev/null +++ b/libgo/go/regexp/testdata/repetition.dat @@ -0,0 +1,163 @@ +NOTE implicit vs. explicit repetitions : 2009-02-02 + +# Glenn Fowler <gsf@research.att.com> +# conforming matches (column 4) must match one of the following BREs +# NOMATCH +# (0,.)\((\(.\),\(.\))(?,?)(\2,\3)\)* +# (0,.)\((\(.\),\(.\))(\2,\3)(?,?)\)* +# i.e., each 3-tuple has two identical elements and one (?,?) + +E ((..)|(.)) NULL NOMATCH +E ((..)|(.))((..)|(.)) NULL NOMATCH +E ((..)|(.))((..)|(.))((..)|(.)) NULL NOMATCH + +E ((..)|(.)){1} NULL NOMATCH +E ((..)|(.)){2} NULL NOMATCH +E ((..)|(.)){3} NULL NOMATCH + +E ((..)|(.))* NULL (0,0) + +E ((..)|(.)) a (0,1)(0,1)(?,?)(0,1) +E ((..)|(.))((..)|(.)) a NOMATCH +E ((..)|(.))((..)|(.))((..)|(.)) a NOMATCH + +E ((..)|(.)){1} a (0,1)(0,1)(?,?)(0,1) +E ((..)|(.)){2} a NOMATCH +E ((..)|(.)){3} a NOMATCH + +E ((..)|(.))* a (0,1)(0,1)(?,?)(0,1) + +E ((..)|(.)) aa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.))((..)|(.)) aa (0,2)(0,1)(?,?)(0,1)(1,2)(?,?)(1,2) +E ((..)|(.))((..)|(.))((..)|(.)) aa NOMATCH + +E ((..)|(.)){1} aa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.)){2} aa (0,2)(1,2)(?,?)(1,2) +E ((..)|(.)){3} aa NOMATCH + +E ((..)|(.))* aa (0,2)(0,2)(0,2)(?,?) + +E ((..)|(.)) aaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.))((..)|(.)) aaa (0,3)(0,2)(0,2)(?,?)(2,3)(?,?)(2,3) +E ((..)|(.))((..)|(.))((..)|(.)) aaa (0,3)(0,1)(?,?)(0,1)(1,2)(?,?)(1,2)(2,3)(?,?)(2,3) + +E ((..)|(.)){1} aaa (0,2)(0,2)(0,2)(?,?) +#E ((..)|(.)){2} aaa (0,3)(2,3)(?,?)(2,3) +E ((..)|(.)){2} aaa (0,3)(2,3)(0,2)(2,3) RE2/Go +E ((..)|(.)){3} aaa (0,3)(2,3)(?,?)(2,3) + +#E ((..)|(.))* aaa (0,3)(2,3)(?,?)(2,3) +E ((..)|(.))* aaa (0,3)(2,3)(0,2)(2,3) RE2/Go + +E ((..)|(.)) aaaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.))((..)|(.)) aaaa (0,4)(0,2)(0,2)(?,?)(2,4)(2,4)(?,?) +E ((..)|(.))((..)|(.))((..)|(.)) aaaa (0,4)(0,2)(0,2)(?,?)(2,3)(?,?)(2,3)(3,4)(?,?)(3,4) + +E ((..)|(.)){1} aaaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.)){2} aaaa (0,4)(2,4)(2,4)(?,?) +#E ((..)|(.)){3} aaaa (0,4)(3,4)(?,?)(3,4) +E ((..)|(.)){3} aaaa (0,4)(3,4)(0,2)(3,4) RE2/Go + +E ((..)|(.))* aaaa (0,4)(2,4)(2,4)(?,?) + +E ((..)|(.)) aaaaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.))((..)|(.)) aaaaa (0,4)(0,2)(0,2)(?,?)(2,4)(2,4)(?,?) +E ((..)|(.))((..)|(.))((..)|(.)) aaaaa (0,5)(0,2)(0,2)(?,?)(2,4)(2,4)(?,?)(4,5)(?,?)(4,5) + +E ((..)|(.)){1} aaaaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.)){2} aaaaa (0,4)(2,4)(2,4)(?,?) +#E ((..)|(.)){3} aaaaa (0,5)(4,5)(?,?)(4,5) +E ((..)|(.)){3} aaaaa (0,5)(4,5)(2,4)(4,5) RE2/Go + +#E ((..)|(.))* aaaaa (0,5)(4,5)(?,?)(4,5) +E ((..)|(.))* aaaaa (0,5)(4,5)(2,4)(4,5) RE2/Go + +E ((..)|(.)) aaaaaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.))((..)|(.)) aaaaaa (0,4)(0,2)(0,2)(?,?)(2,4)(2,4)(?,?) +E ((..)|(.))((..)|(.))((..)|(.)) aaaaaa (0,6)(0,2)(0,2)(?,?)(2,4)(2,4)(?,?)(4,6)(4,6)(?,?) + +E ((..)|(.)){1} aaaaaa (0,2)(0,2)(0,2)(?,?) +E ((..)|(.)){2} aaaaaa (0,4)(2,4)(2,4)(?,?) +E ((..)|(.)){3} aaaaaa (0,6)(4,6)(4,6)(?,?) + +E ((..)|(.))* aaaaaa (0,6)(4,6)(4,6)(?,?) + +NOTE additional repetition tests graciously provided by Chris Kuklewicz www.haskell.org 2009-02-02 + +# These test a bug in OS X / FreeBSD / NetBSD, and libtree. +# Linux/GLIBC gets the {8,} and {8,8} wrong. + +:HA#100:E X(.?){0,}Y X1234567Y (0,9)(7,8) +:HA#101:E X(.?){1,}Y X1234567Y (0,9)(7,8) +:HA#102:E X(.?){2,}Y X1234567Y (0,9)(7,8) +:HA#103:E X(.?){3,}Y X1234567Y (0,9)(7,8) +:HA#104:E X(.?){4,}Y X1234567Y (0,9)(7,8) +:HA#105:E X(.?){5,}Y X1234567Y (0,9)(7,8) +:HA#106:E X(.?){6,}Y X1234567Y (0,9)(7,8) +:HA#107:E X(.?){7,}Y X1234567Y (0,9)(7,8) +:HA#108:E X(.?){8,}Y X1234567Y (0,9)(8,8) +#:HA#110:E X(.?){0,8}Y X1234567Y (0,9)(7,8) +:HA#110:E X(.?){0,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#111:E X(.?){1,8}Y X1234567Y (0,9)(7,8) +:HA#111:E X(.?){1,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#112:E X(.?){2,8}Y X1234567Y (0,9)(7,8) +:HA#112:E X(.?){2,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#113:E X(.?){3,8}Y X1234567Y (0,9)(7,8) +:HA#113:E X(.?){3,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#114:E X(.?){4,8}Y X1234567Y (0,9)(7,8) +:HA#114:E X(.?){4,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#115:E X(.?){5,8}Y X1234567Y (0,9)(7,8) +:HA#115:E X(.?){5,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#116:E X(.?){6,8}Y X1234567Y (0,9)(7,8) +:HA#116:E X(.?){6,8}Y X1234567Y (0,9)(8,8) RE2/Go +#:HA#117:E X(.?){7,8}Y X1234567Y (0,9)(7,8) +:HA#117:E X(.?){7,8}Y X1234567Y (0,9)(8,8) RE2/Go +:HA#118:E X(.?){8,8}Y X1234567Y (0,9)(8,8) + +# These test a fixed bug in my regex-tdfa that did not keep the expanded +# form properly grouped, so right association did the wrong thing with +# these ambiguous patterns (crafted just to test my code when I became +# suspicious of my implementation). The first subexpression should use +# "ab" then "a" then "bcd". + +# OS X / FreeBSD / NetBSD badly fail many of these, with impossible +# results like (0,6)(4,5)(6,6). + +:HA#260:E (a|ab|c|bcd){0,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#261:E (a|ab|c|bcd){1,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#262:E (a|ab|c|bcd){2,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#263:E (a|ab|c|bcd){3,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#264:E (a|ab|c|bcd){4,}(d*) ababcd NOMATCH +:HA#265:E (a|ab|c|bcd){0,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#266:E (a|ab|c|bcd){1,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#267:E (a|ab|c|bcd){2,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#268:E (a|ab|c|bcd){3,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#269:E (a|ab|c|bcd){4,10}(d*) ababcd NOMATCH +:HA#270:E (a|ab|c|bcd)*(d*) ababcd (0,6)(3,6)(6,6) +:HA#271:E (a|ab|c|bcd)+(d*) ababcd (0,6)(3,6)(6,6) + +# The above worked on Linux/GLIBC but the following often fail. +# They also trip up OS X / FreeBSD / NetBSD: + +#:HA#280:E (ab|a|c|bcd){0,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#280:E (ab|a|c|bcd){0,}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#281:E (ab|a|c|bcd){1,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#281:E (ab|a|c|bcd){1,}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#282:E (ab|a|c|bcd){2,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#282:E (ab|a|c|bcd){2,}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#283:E (ab|a|c|bcd){3,}(d*) ababcd (0,6)(3,6)(6,6) +:HA#283:E (ab|a|c|bcd){3,}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +:HA#284:E (ab|a|c|bcd){4,}(d*) ababcd NOMATCH +#:HA#285:E (ab|a|c|bcd){0,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#285:E (ab|a|c|bcd){0,10}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#286:E (ab|a|c|bcd){1,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#286:E (ab|a|c|bcd){1,10}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#287:E (ab|a|c|bcd){2,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#287:E (ab|a|c|bcd){2,10}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#288:E (ab|a|c|bcd){3,10}(d*) ababcd (0,6)(3,6)(6,6) +:HA#288:E (ab|a|c|bcd){3,10}(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +:HA#289:E (ab|a|c|bcd){4,10}(d*) ababcd NOMATCH +#:HA#290:E (ab|a|c|bcd)*(d*) ababcd (0,6)(3,6)(6,6) +:HA#290:E (ab|a|c|bcd)*(d*) ababcd (0,6)(4,5)(5,6) RE2/Go +#:HA#291:E (ab|a|c|bcd)+(d*) ababcd (0,6)(3,6)(6,6) +:HA#291:E (ab|a|c|bcd)+(d*) ababcd (0,6)(4,5)(5,6) RE2/Go diff --git a/libgo/go/regexp/testdata/testregex.c b/libgo/go/regexp/testdata/testregex.c new file mode 100644 index 0000000..37545d0 --- /dev/null +++ b/libgo/go/regexp/testdata/testregex.c @@ -0,0 +1,2286 @@ +#pragma prototyped noticed + +/* + * regex(3) test harness + * + * build: cc -o testregex testregex.c + * help: testregex --man + * note: REG_* features are detected by #ifdef; if REG_* are enums + * then supply #define REG_foo REG_foo for each enum REG_foo + * + * Glenn Fowler <gsf@research.att.com> + * AT&T Research + * + * PLEASE: publish your tests so everyone can benefit + * + * The following license covers testregex.c and all associated test data. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of THIS SOFTWARE FILE (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following disclaimer: + * + * THIS SOFTWARE IS PROVIDED BY AT&T ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL AT&T BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +static const char id[] = "\n@(#)$Id: testregex (AT&T Research) 2010-06-10 $\0\n"; + +#if _PACKAGE_ast +#include <ast.h> +#else +#include <sys/types.h> +#endif + +#include <stdio.h> +#include <regex.h> +#include <ctype.h> +#include <setjmp.h> +#include <signal.h> +#include <string.h> +#include <unistd.h> + +#ifdef __STDC__ +#include <stdlib.h> +#include <locale.h> +#endif + +#ifndef RE_DUP_MAX +#define RE_DUP_MAX 32767 +#endif + +#if !_PACKAGE_ast +#undef REG_DISCIPLINE +#endif + +#ifndef REG_DELIMITED +#undef _REG_subcomp +#endif + +#define TEST_ARE 0x00000001 +#define TEST_BRE 0x00000002 +#define TEST_ERE 0x00000004 +#define TEST_KRE 0x00000008 +#define TEST_LRE 0x00000010 +#define TEST_SRE 0x00000020 + +#define TEST_EXPAND 0x00000100 +#define TEST_LENIENT 0x00000200 + +#define TEST_QUERY 0x00000400 +#define TEST_SUB 0x00000800 +#define TEST_UNSPECIFIED 0x00001000 +#define TEST_VERIFY 0x00002000 +#define TEST_AND 0x00004000 +#define TEST_OR 0x00008000 + +#define TEST_DELIMIT 0x00010000 +#define TEST_OK 0x00020000 +#define TEST_SAME 0x00040000 + +#define TEST_ACTUAL 0x00100000 +#define TEST_BASELINE 0x00200000 +#define TEST_FAIL 0x00400000 +#define TEST_PASS 0x00800000 +#define TEST_SUMMARY 0x01000000 + +#define TEST_IGNORE_ERROR 0x02000000 +#define TEST_IGNORE_OVER 0x04000000 +#define TEST_IGNORE_POSITION 0x08000000 + +#define TEST_CATCH 0x10000000 +#define TEST_VERBOSE 0x20000000 + +#define TEST_DECOMP 0x40000000 + +#define TEST_GLOBAL (TEST_ACTUAL|TEST_AND|TEST_BASELINE|TEST_CATCH|TEST_FAIL|TEST_IGNORE_ERROR|TEST_IGNORE_OVER|TEST_IGNORE_POSITION|TEST_OR|TEST_PASS|TEST_SUMMARY|TEST_VERBOSE) + +#ifdef REG_DISCIPLINE + + +#include <stk.h> + +typedef struct Disc_s +{ + regdisc_t disc; + int ordinal; + Sfio_t* sp; +} Disc_t; + +static void* +compf(const regex_t* re, const char* xstr, size_t xlen, regdisc_t* disc) +{ + Disc_t* dp = (Disc_t*)disc; + + return (void*)((char*)0 + ++dp->ordinal); +} + +static int +execf(const regex_t* re, void* data, const char* xstr, size_t xlen, const char* sstr, size_t slen, char** snxt, regdisc_t* disc) +{ + Disc_t* dp = (Disc_t*)disc; + + sfprintf(dp->sp, "{%-.*s}(%lu:%d)", xlen, xstr, (char*)data - (char*)0, slen); + return atoi(xstr); +} + +static void* +resizef(void* handle, void* data, size_t size) +{ + if (!size) + return 0; + return stkalloc((Sfio_t*)handle, size); +} + +#endif + +#ifndef NiL +#ifdef __STDC__ +#define NiL 0 +#else +#define NiL (char*)0 +#endif +#endif + +#define H(x) do{if(html)fprintf(stderr,x);}while(0) +#define T(x) fprintf(stderr,x) + +static void +help(int html) +{ +H("<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n"); +H("<HTML>\n"); +H("<HEAD>\n"); +H("<TITLE>testregex man document</TITLE>\n"); +H("</HEAD>\n"); +H("<BODY bgcolor=white>\n"); +H("<PRE>\n"); +T("NAME\n"); +T(" testregex - regex(3) test harness\n"); +T("\n"); +T("SYNOPSIS\n"); +T(" testregex [ options ]\n"); +T("\n"); +T("DESCRIPTION\n"); +T(" testregex reads regex(3) test specifications, one per line, from the\n"); +T(" standard input and writes one output line for each failed test. A\n"); +T(" summary line is written after all tests are done. Each successful\n"); +T(" test is run again with REG_NOSUB. Unsupported features are noted\n"); +T(" before the first test, and tests requiring these features are\n"); +T(" silently ignored.\n"); +T("\n"); +T("OPTIONS\n"); +T(" -c catch signals and non-terminating calls\n"); +T(" -e ignore error return mismatches\n"); +T(" -h list help on standard error\n"); +T(" -n do not repeat successful tests with regnexec()\n"); +T(" -o ignore match[] overrun errors\n"); +T(" -p ignore negative position mismatches\n"); +T(" -s use stack instead of malloc\n"); +T(" -x do not repeat successful tests with REG_NOSUB\n"); +T(" -v list each test line\n"); +T(" -A list failed test lines with actual answers\n"); +T(" -B list all test lines with actual answers\n"); +T(" -F list failed test lines\n"); +T(" -P list passed test lines\n"); +T(" -S output one summary line\n"); +T("\n"); +T("INPUT FORMAT\n"); +T(" Input lines may be blank, a comment beginning with #, or a test\n"); +T(" specification. A specification is five fields separated by one\n"); +T(" or more tabs. NULL denotes the empty string and NIL denotes the\n"); +T(" 0 pointer.\n"); +T("\n"); +T(" Field 1: the regex(3) flags to apply, one character per REG_feature\n"); +T(" flag. The test is skipped if REG_feature is not supported by the\n"); +T(" implementation. If the first character is not [BEASKLP] then the\n"); +T(" specification is a global control line. One or more of [BEASKLP] may be\n"); +T(" specified; the test will be repeated for each mode.\n"); +T("\n"); +T(" B basic BRE (grep, ed, sed)\n"); +T(" E REG_EXTENDED ERE (egrep)\n"); +T(" A REG_AUGMENTED ARE (egrep with negation)\n"); +T(" S REG_SHELL SRE (sh glob)\n"); +T(" K REG_SHELL|REG_AUGMENTED KRE (ksh glob)\n"); +T(" L REG_LITERAL LRE (fgrep)\n"); +T("\n"); +T(" a REG_LEFT|REG_RIGHT implicit ^...$\n"); +T(" b REG_NOTBOL lhs does not match ^\n"); +T(" c REG_COMMENT ignore space and #...\\n\n"); +T(" d REG_SHELL_DOT explicit leading . match\n"); +T(" e REG_NOTEOL rhs does not match $\n"); +T(" f REG_MULTIPLE multiple \\n separated patterns\n"); +T(" g FNM_LEADING_DIR testfnmatch only -- match until /\n"); +T(" h REG_MULTIREF multiple digit backref\n"); +T(" i REG_ICASE ignore case\n"); +T(" j REG_SPAN . matches \\n\n"); +T(" k REG_ESCAPE \\ to ecape [...] delimiter\n"); +T(" l REG_LEFT implicit ^...\n"); +T(" m REG_MINIMAL minimal match\n"); +T(" n REG_NEWLINE explicit \\n match\n"); +T(" o REG_ENCLOSED (|&) magic inside [@|&](...)\n"); +T(" p REG_SHELL_PATH explicit / match\n"); +T(" q REG_DELIMITED delimited pattern\n"); +T(" r REG_RIGHT implicit ...$\n"); +T(" s REG_SHELL_ESCAPED \\ not special\n"); +T(" t REG_MUSTDELIM all delimiters must be specified\n"); +T(" u standard unspecified behavior -- errors not counted\n"); +T(" v REG_CLASS_ESCAPE \\ special inside [...]\n"); +T(" w REG_NOSUB no subexpression match array\n"); +T(" x REG_LENIENT let some errors slide\n"); +T(" y REG_LEFT regexec() implicit ^...\n"); +T(" z REG_NULL NULL subexpressions ok\n"); +T(" $ expand C \\c escapes in fields 2 and 3\n"); +T(" / field 2 is a regsubcomp() expression\n"); +T(" = field 3 is a regdecomp() expression\n"); +T("\n"); +T(" Field 1 control lines:\n"); +T("\n"); +T(" C set LC_COLLATE and LC_CTYPE to locale in field 2\n"); +T("\n"); +T(" ?test ... output field 5 if passed and != EXPECTED, silent otherwise\n"); +T(" &test ... output field 5 if current and previous passed\n"); +T(" |test ... output field 5 if current passed and previous failed\n"); +T(" ; ... output field 2 if previous failed\n"); +T(" {test ... skip if failed until }\n"); +T(" } end of skip\n"); +T("\n"); +T(" : comment comment copied as output NOTE\n"); +T(" :comment:test :comment: ignored\n"); +T(" N[OTE] comment comment copied as output NOTE\n"); +T(" T[EST] comment comment\n"); +T("\n"); +T(" number use number for nmatch (20 by default)\n"); +T("\n"); +T(" Field 2: the regular expression pattern; SAME uses the pattern from\n"); +T(" the previous specification. RE_DUP_MAX inside {...} expands to the\n"); +T(" value from <limits.h>.\n"); +T("\n"); +T(" Field 3: the string to match. X...{RE_DUP_MAX} expands to RE_DUP_MAX\n"); +T(" copies of X.\n"); +T("\n"); +T(" Field 4: the test outcome. This is either one of the posix error\n"); +T(" codes (with REG_ omitted) or the match array, a list of (m,n)\n"); +T(" entries with m and n being first and last+1 positions in the\n"); +T(" field 3 string, or NULL if REG_NOSUB is in effect and success\n"); +T(" is expected. BADPAT is acceptable in place of any regcomp(3)\n"); +T(" error code. The match[] array is initialized to (-2,-2) before\n"); +T(" each test. All array elements from 0 to nmatch-1 must be specified\n"); +T(" in the outcome. Unspecified endpoints (offset -1) are denoted by ?.\n"); +T(" Unset endpoints (offset -2) are denoted by X. {x}(o:n) denotes a\n"); +T(" matched (?{...}) expression, where x is the text enclosed by {...},\n"); +T(" o is the expression ordinal counting from 1, and n is the length of\n"); +T(" the unmatched portion of the subject string. If x starts with a\n"); +T(" number then that is the return value of re_execf(), otherwise 0 is\n"); +T(" returned. RE_DUP_MAX[-+]N expands to the <limits.h> value -+N.\n"); +T("\n"); +T(" Field 5: optional comment appended to the report.\n"); +T("\n"); +T("CAVEAT\n"); +T(" If a regex implementation misbehaves with memory then all bets are off.\n"); +T("\n"); +T("CONTRIBUTORS\n"); +T(" Glenn Fowler gsf@research.att.com (ksh strmatch, regex extensions)\n"); +T(" David Korn dgk@research.att.com (ksh glob matcher)\n"); +T(" Doug McIlroy mcilroy@dartmouth.edu (ast regex/testre in C++)\n"); +T(" Tom Lord lord@regexps.com (rx tests)\n"); +T(" Henry Spencer henry@zoo.toronto.edu (original public regex)\n"); +T(" Andrew Hume andrew@research.att.com (gre tests)\n"); +T(" John Maddock John_Maddock@compuserve.com (regex++ tests)\n"); +T(" Philip Hazel ph10@cam.ac.uk (pcre tests)\n"); +T(" Ville Laurikari vl@iki.fi (libtre tests)\n"); +H("</PRE>\n"); +H("</BODY>\n"); +H("</HTML>\n"); +} + +#ifndef elementsof +#define elementsof(x) (sizeof(x)/sizeof(x[0])) +#endif + +#ifndef streq +#define streq(a,b) (*(a)==*(b)&&!strcmp(a,b)) +#endif + +#define HUNG 2 +#define NOTEST (~0) + +#ifndef REG_TEST_DEFAULT +#define REG_TEST_DEFAULT 0 +#endif + +#ifndef REG_EXEC_DEFAULT +#define REG_EXEC_DEFAULT 0 +#endif + +static const char* unsupported[] = +{ + "BASIC", +#ifndef REG_EXTENDED + "EXTENDED", +#endif +#ifndef REG_AUGMENTED + "AUGMENTED", +#endif +#ifndef REG_SHELL + "SHELL", +#endif + +#ifndef REG_CLASS_ESCAPE + "CLASS_ESCAPE", +#endif +#ifndef REG_COMMENT + "COMMENT", +#endif +#ifndef REG_DELIMITED + "DELIMITED", +#endif +#ifndef REG_DISCIPLINE + "DISCIPLINE", +#endif +#ifndef REG_ESCAPE + "ESCAPE", +#endif +#ifndef REG_ICASE + "ICASE", +#endif +#ifndef REG_LEFT + "LEFT", +#endif +#ifndef REG_LENIENT + "LENIENT", +#endif +#ifndef REG_LITERAL + "LITERAL", +#endif +#ifndef REG_MINIMAL + "MINIMAL", +#endif +#ifndef REG_MULTIPLE + "MULTIPLE", +#endif +#ifndef REG_MULTIREF + "MULTIREF", +#endif +#ifndef REG_MUSTDELIM + "MUSTDELIM", +#endif +#ifndef REG_NEWLINE + "NEWLINE", +#endif +#ifndef REG_NOTBOL + "NOTBOL", +#endif +#ifndef REG_NOTEOL + "NOTEOL", +#endif +#ifndef REG_NULL + "NULL", +#endif +#ifndef REG_RIGHT + "RIGHT", +#endif +#ifndef REG_SHELL_DOT + "SHELL_DOT", +#endif +#ifndef REG_SHELL_ESCAPED + "SHELL_ESCAPED", +#endif +#ifndef REG_SHELL_GROUP + "SHELL_GROUP", +#endif +#ifndef REG_SHELL_PATH + "SHELL_PATH", +#endif +#ifndef REG_SPAN + "SPAN", +#endif +#if REG_NOSUB & REG_TEST_DEFAULT + "SUBMATCH", +#endif +#if !_REG_nexec + "regnexec", +#endif +#if !_REG_subcomp + "regsubcomp", +#endif +#if !_REG_decomp + "redecomp", +#endif + 0 +}; + +#ifndef REG_CLASS_ESCAPE +#define REG_CLASS_ESCAPE NOTEST +#endif +#ifndef REG_COMMENT +#define REG_COMMENT NOTEST +#endif +#ifndef REG_DELIMITED +#define REG_DELIMITED NOTEST +#endif +#ifndef REG_ESCAPE +#define REG_ESCAPE NOTEST +#endif +#ifndef REG_ICASE +#define REG_ICASE NOTEST +#endif +#ifndef REG_LEFT +#define REG_LEFT NOTEST +#endif +#ifndef REG_LENIENT +#define REG_LENIENT 0 +#endif +#ifndef REG_MINIMAL +#define REG_MINIMAL NOTEST +#endif +#ifndef REG_MULTIPLE +#define REG_MULTIPLE NOTEST +#endif +#ifndef REG_MULTIREF +#define REG_MULTIREF NOTEST +#endif +#ifndef REG_MUSTDELIM +#define REG_MUSTDELIM NOTEST +#endif +#ifndef REG_NEWLINE +#define REG_NEWLINE NOTEST +#endif +#ifndef REG_NOTBOL +#define REG_NOTBOL NOTEST +#endif +#ifndef REG_NOTEOL +#define REG_NOTEOL NOTEST +#endif +#ifndef REG_NULL +#define REG_NULL NOTEST +#endif +#ifndef REG_RIGHT +#define REG_RIGHT NOTEST +#endif +#ifndef REG_SHELL_DOT +#define REG_SHELL_DOT NOTEST +#endif +#ifndef REG_SHELL_ESCAPED +#define REG_SHELL_ESCAPED NOTEST +#endif +#ifndef REG_SHELL_GROUP +#define REG_SHELL_GROUP NOTEST +#endif +#ifndef REG_SHELL_PATH +#define REG_SHELL_PATH NOTEST +#endif +#ifndef REG_SPAN +#define REG_SPAN NOTEST +#endif + +#define REG_UNKNOWN (-1) + +#ifndef REG_ENEWLINE +#define REG_ENEWLINE (REG_UNKNOWN-1) +#endif +#ifndef REG_ENULL +#ifndef REG_EMPTY +#define REG_ENULL (REG_UNKNOWN-2) +#else +#define REG_ENULL REG_EMPTY +#endif +#endif +#ifndef REG_ECOUNT +#define REG_ECOUNT (REG_UNKNOWN-3) +#endif +#ifndef REG_BADESC +#define REG_BADESC (REG_UNKNOWN-4) +#endif +#ifndef REG_EMEM +#define REG_EMEM (REG_UNKNOWN-5) +#endif +#ifndef REG_EHUNG +#define REG_EHUNG (REG_UNKNOWN-6) +#endif +#ifndef REG_EBUS +#define REG_EBUS (REG_UNKNOWN-7) +#endif +#ifndef REG_EFAULT +#define REG_EFAULT (REG_UNKNOWN-8) +#endif +#ifndef REG_EFLAGS +#define REG_EFLAGS (REG_UNKNOWN-9) +#endif +#ifndef REG_EDELIM +#define REG_EDELIM (REG_UNKNOWN-9) +#endif + +static const struct { int code; char* name; } codes[] = +{ + REG_UNKNOWN, "UNKNOWN", + REG_NOMATCH, "NOMATCH", + REG_BADPAT, "BADPAT", + REG_ECOLLATE, "ECOLLATE", + REG_ECTYPE, "ECTYPE", + REG_EESCAPE, "EESCAPE", + REG_ESUBREG, "ESUBREG", + REG_EBRACK, "EBRACK", + REG_EPAREN, "EPAREN", + REG_EBRACE, "EBRACE", + REG_BADBR, "BADBR", + REG_ERANGE, "ERANGE", + REG_ESPACE, "ESPACE", + REG_BADRPT, "BADRPT", + REG_ENEWLINE, "ENEWLINE", + REG_ENULL, "ENULL", + REG_ECOUNT, "ECOUNT", + REG_BADESC, "BADESC", + REG_EMEM, "EMEM", + REG_EHUNG, "EHUNG", + REG_EBUS, "EBUS", + REG_EFAULT, "EFAULT", + REG_EFLAGS, "EFLAGS", + REG_EDELIM, "EDELIM", +}; + +static struct +{ + regmatch_t NOMATCH; + int errors; + int extracted; + int ignored; + int lineno; + int passed; + int signals; + int unspecified; + int verify; + int warnings; + char* file; + char* stack; + char* which; + jmp_buf gotcha; +#ifdef REG_DISCIPLINE + Disc_t disc; +#endif +} state; + +static void +quote(char* s, int len, unsigned long test) +{ + unsigned char* u = (unsigned char*)s; + unsigned char* e; + int c; +#ifdef MB_CUR_MAX + int w; +#endif + + if (!u) + printf("NIL"); + else if (!*u && len <= 1) + printf("NULL"); + else if (test & TEST_EXPAND) + { + if (len < 0) + len = strlen((char*)u); + e = u + len; + if (test & TEST_DELIMIT) + printf("\""); + while (u < e) + switch (c = *u++) + { + case '\\': + printf("\\\\"); + break; + case '"': + if (test & TEST_DELIMIT) + printf("\\\""); + else + printf("\""); + break; + case '\a': + printf("\\a"); + break; + case '\b': + printf("\\b"); + break; + case 033: + printf("\\e"); + break; + case '\f': + printf("\\f"); + break; + case '\n': + printf("\\n"); + break; + case '\r': + printf("\\r"); + break; + case '\t': + printf("\\t"); + break; + case '\v': + printf("\\v"); + break; + default: +#ifdef MB_CUR_MAX + s = (char*)u - 1; + if ((w = mblen(s, (char*)e - s)) > 1) + { + u += w - 1; + fwrite(s, 1, w, stdout); + } + else +#endif + if (!iscntrl(c) && isprint(c)) + putchar(c); + else + printf("\\x%02x", c); + break; + } + if (test & TEST_DELIMIT) + printf("\""); + } + else + printf("%s", s); +} + +static void +report(char* comment, char* fun, char* re, char* s, int len, char* msg, int flags, unsigned long test) +{ + if (state.file) + printf("%s:", state.file); + printf("%d:", state.lineno); + if (re) + { + printf(" "); + quote(re, -1, test|TEST_DELIMIT); + if (s) + { + printf(" versus "); + quote(s, len, test|TEST_DELIMIT); + } + } + if (test & TEST_UNSPECIFIED) + { + state.unspecified++; + printf(" unspecified behavior"); + } + else + state.errors++; + if (state.which) + printf(" %s", state.which); + if (flags & REG_NOSUB) + printf(" NOSUB"); + if (fun) + printf(" %s", fun); + if (comment[strlen(comment)-1] == '\n') + printf(" %s", comment); + else + { + printf(" %s: ", comment); + if (msg) + printf("%s: ", msg); + } +} + +static void +error(regex_t* preg, int code) +{ + char* msg; + char buf[256]; + + switch (code) + { + case REG_EBUS: + msg = "bus error"; + break; + case REG_EFAULT: + msg = "memory fault"; + break; + case REG_EHUNG: + msg = "did not terminate"; + break; + default: + regerror(code, preg, msg = buf, sizeof buf); + break; + } + printf("%s\n", msg); +} + +static void +bad(char* comment, char* re, char* s, int len, unsigned long test) +{ + printf("bad test case "); + report(comment, NiL, re, s, len, NiL, 0, test); + exit(1); +} + +static int +escape(char* s) +{ + char* b; + char* t; + char* q; + char* e; + int c; + + for (b = t = s; *t = *s; s++, t++) + if (*s == '\\') + switch (*++s) + { + case '\\': + break; + case 'a': + *t = '\a'; + break; + case 'b': + *t = '\b'; + break; + case 'c': + if (*t = *++s) + *t &= 037; + else + s--; + break; + case 'e': + case 'E': + *t = 033; + break; + case 'f': + *t = '\f'; + break; + case 'n': + *t = '\n'; + break; + case 'r': + *t = '\r'; + break; + case 's': + *t = ' '; + break; + case 't': + *t = '\t'; + break; + case 'v': + *t = '\v'; + break; + case 'u': + case 'x': + c = 0; + q = c == 'u' ? (s + 5) : (char*)0; + e = s + 1; + while (!e || !q || s < q) + { + switch (*++s) + { + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + c = (c << 4) + *s - 'a' + 10; + continue; + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + c = (c << 4) + *s - 'A' + 10; + continue; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c = (c << 4) + *s - '0'; + continue; + case '{': + case '[': + if (s != e) + { + s--; + break; + } + e = 0; + continue; + case '}': + case ']': + if (e) + s--; + break; + default: + s--; + break; + } + break; + } + *t = c; + break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + c = *s - '0'; + q = s + 2; + while (s < q) + { + switch (*++s) + { + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + c = (c << 3) + *s - '0'; + break; + default: + q = --s; + break; + } + } + *t = c; + break; + default: + *(s + 1) = 0; + bad("invalid C \\ escape\n", s - 1, NiL, 0, 0); + } + return t - b; +} + +static void +matchoffprint(int off) +{ + switch (off) + { + case -2: + printf("X"); + break; + case -1: + printf("?"); + break; + default: + printf("%d", off); + break; + } +} + +static void +matchprint(regmatch_t* match, int nmatch, int nsub, char* ans, unsigned long test) +{ + int i; + + for (; nmatch > nsub + 1; nmatch--) + if ((match[nmatch-1].rm_so != -1 || match[nmatch-1].rm_eo != -1) && (!(test & TEST_IGNORE_POSITION) || match[nmatch-1].rm_so >= 0 && match[nmatch-1].rm_eo >= 0)) + break; + for (i = 0; i < nmatch; i++) + { + printf("("); + matchoffprint(match[i].rm_so); + printf(","); + matchoffprint(match[i].rm_eo); + printf(")"); + } + if (!(test & (TEST_ACTUAL|TEST_BASELINE))) + { + if (ans) + printf(" expected: %s", ans); + printf("\n"); + } +} + +static int +matchcheck(regmatch_t* match, int nmatch, int nsub, char* ans, char* re, char* s, int len, int flags, unsigned long test) +{ + char* p; + int i; + int m; + int n; + + if (streq(ans, "OK")) + return test & (TEST_BASELINE|TEST_PASS|TEST_VERIFY); + for (i = 0, p = ans; i < nmatch && *p; i++) + { + if (*p == '{') + { +#ifdef REG_DISCIPLINE + char* x; + + if (!(x = sfstruse(state.disc.sp))) + bad("out of space [discipline string]\n", NiL, NiL, 0, 0); + if (strcmp(p, x)) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + return 0; + report("callout failed", NiL, re, s, len, NiL, flags, test); + quote(p, -1, test); + printf(" expected, "); + quote(x, -1, test); + printf(" returned\n"); + } +#endif + break; + } + if (*p++ != '(') + bad("improper answer\n", re, s, -1, test); + if (*p == '?') + { + m = -1; + p++; + } + else if (*p == 'R' && !memcmp(p, "RE_DUP_MAX", 10)) + { + m = RE_DUP_MAX; + p += 10; + if (*p == '+' || *p == '-') + m += strtol(p, &p, 10); + } + else + m = strtol(p, &p, 10); + if (*p++ != ',') + bad("improper answer\n", re, s, -1, test); + if (*p == '?') + { + n = -1; + p++; + } + else if (*p == 'R' && !memcmp(p, "RE_DUP_MAX", 10)) + { + n = RE_DUP_MAX; + p += 10; + if (*p == '+' || *p == '-') + n += strtol(p, &p, 10); + } + else + n = strtol(p, &p, 10); + if (*p++ != ')') + bad("improper answer\n", re, s, -1, test); + if (m!=match[i].rm_so || n!=match[i].rm_eo) + { + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY))) + { + report("failed: match was", NiL, re, s, len, NiL, flags, test); + matchprint(match, nmatch, nsub, ans, test); + } + return 0; + } + } + for (; i < nmatch; i++) + { + if (match[i].rm_so!=-1 || match[i].rm_eo!=-1) + { + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_VERIFY))) + { + if ((test & TEST_IGNORE_POSITION) && (match[i].rm_so<0 || match[i].rm_eo<0)) + { + state.ignored++; + return 0; + } + if (!(test & TEST_SUMMARY)) + { + report("failed: match was", NiL, re, s, len, NiL, flags, test); + matchprint(match, nmatch, nsub, ans, test); + } + } + return 0; + } + } + if (!(test & TEST_IGNORE_OVER) && match[nmatch].rm_so != state.NOMATCH.rm_so) + { + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY))) + { + report("failed: overran match array", NiL, re, s, len, NiL, flags, test); + matchprint(match, nmatch + 1, nsub, NiL, test); + } + return 0; + } + return 1; +} + +static void +sigunblock(int s) +{ +#ifdef SIG_SETMASK + int op; + sigset_t mask; + + sigemptyset(&mask); + if (s) + { + sigaddset(&mask, s); + op = SIG_UNBLOCK; + } + else op = SIG_SETMASK; + sigprocmask(op, &mask, NiL); +#else +#ifdef sigmask + sigsetmask(s ? (sigsetmask(0L) & ~sigmask(s)) : 0L); +#endif +#endif +} + +static void +gotcha(int sig) +{ + int ret; + + signal(sig, gotcha); + alarm(0); + state.signals++; + switch (sig) + { + case SIGALRM: + ret = REG_EHUNG; + break; + case SIGBUS: + ret = REG_EBUS; + break; + default: + ret = REG_EFAULT; + break; + } + sigunblock(sig); + longjmp(state.gotcha, ret); +} + +static char* +getline(FILE* fp) +{ + static char buf[32 * 1024]; + + register char* s = buf; + register char* e = &buf[sizeof(buf)]; + register char* b; + + for (;;) + { + if (!(b = fgets(s, e - s, fp))) + return 0; + state.lineno++; + s += strlen(s); + if (s == b || *--s != '\n' || s == b || *(s - 1) != '\\') + { + *s = 0; + break; + } + s--; + } + return buf; +} + +static unsigned long +note(unsigned long level, char* msg, unsigned long skip, unsigned long test) +{ + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_SUMMARY)) && !skip) + { + printf("NOTE\t"); + if (msg) + printf("%s: ", msg); + printf("skipping lines %d", state.lineno); + } + return skip | level; +} + +#define TABS(n) &ts[7-((n)&7)] + +static char ts[] = "\t\t\t\t\t\t\t"; + +static unsigned long +extract(int* tabs, char* spec, char* re, char* s, char* ans, char* msg, char* accept, regmatch_t* match, int nmatch, int nsub, unsigned long skip, unsigned long level, unsigned long test) +{ + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_OK|TEST_PASS|TEST_SUMMARY)) + { + state.extracted = 1; + if (test & TEST_OK) + { + state.passed++; + if ((test & TEST_VERIFY) && !(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_SUMMARY))) + { + if (msg && strcmp(msg, "EXPECTED")) + printf("NOTE\t%s\n", msg); + return skip; + } + test &= ~(TEST_PASS|TEST_QUERY); + } + if (test & (TEST_QUERY|TEST_VERIFY)) + { + if (test & TEST_BASELINE) + test &= ~(TEST_BASELINE|TEST_PASS); + else + test |= TEST_PASS; + skip |= level; + } + if (!(test & TEST_OK)) + { + if (test & TEST_UNSPECIFIED) + state.unspecified++; + else + state.errors++; + } + if (test & (TEST_PASS|TEST_SUMMARY)) + return skip; + test &= ~TEST_DELIMIT; + printf("%s%s", spec, TABS(*tabs++)); + if ((test & (TEST_BASELINE|TEST_SAME)) == (TEST_BASELINE|TEST_SAME)) + printf("SAME"); + else + quote(re, -1, test); + printf("%s", TABS(*tabs++)); + quote(s, -1, test); + printf("%s", TABS(*tabs++)); + if (!(test & (TEST_ACTUAL|TEST_BASELINE)) || !accept && !match) + printf("%s", ans); + else if (accept) + printf("%s", accept); + else + matchprint(match, nmatch, nsub, NiL, test); + if (msg) + printf("%s%s", TABS(*tabs++), msg); + putchar('\n'); + } + else if (test & TEST_QUERY) + skip = note(level, msg, skip, test); + else if (test & TEST_VERIFY) + state.extracted = 1; + return skip; +} + +static int +catchfree(regex_t* preg, int flags, int* tabs, char* spec, char* re, char* s, char* ans, char* msg, char* accept, regmatch_t* match, int nmatch, int nsub, unsigned long skip, unsigned long level, unsigned long test) +{ + int eret; + + if (!(test & TEST_CATCH)) + { + regfree(preg); + eret = 0; + } + else if (!(eret = setjmp(state.gotcha))) + { + alarm(HUNG); + regfree(preg); + alarm(0); + } + else if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + extract(tabs, spec, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test); + else + { + report("failed", "regfree", re, NiL, -1, msg, flags, test); + error(preg, eret); + } + return eret; +} + +static char* +expand(char* os, char* ot) +{ + char* s = os; + char* t; + int n = 0; + int r; + long m; + + for (;;) + { + switch (*s++) + { + case 0: + break; + case '{': + n++; + continue; + case '}': + n--; + continue; + case 'R': + if (n == 1 && !memcmp(s, "E_DUP_MAX", 9)) + { + s--; + for (t = ot; os < s; *t++ = *os++); + r = ((t - ot) >= 5 && t[-1] == '{' && t[-2] == '.' && t[-3] == '.' && t[-4] == '.') ? t[-5] : 0; + os = ot; + m = RE_DUP_MAX; + if (*(s += 10) == '+' || *s == '-') + m += strtol(s, &s, 10); + if (r) + { + t -= 5; + while (m-- > 0) + *t++ = r; + while (*s && *s++ != '}'); + } + else + t += snprintf(t, 32, "%ld", m); + while (*t = *s++) + t++; + break; + } + continue; + default: + continue; + } + break; + } + return os; +} + +int +main(int argc, char** argv) +{ + int flags; + int cflags; + int eflags; + int nmatch; + int nexec; + int nstr; + int cret; + int eret; + int nsub; + int i; + int j; + int expected; + int got; + int locale; + int subunitlen; + int testno; + unsigned long level; + unsigned long skip; + char* p; + char* line; + char* spec; + char* re; + char* s; + char* ans; + char* msg; + char* fun; + char* ppat; + char* subunit; + char* version; + char* field[6]; + char* delim[6]; + FILE* fp; + int tabs[6]; + char unit[64]; + regmatch_t match[100]; + regex_t preg; + + static char pat[32 * 1024]; + static char patbuf[32 * 1024]; + static char strbuf[32 * 1024]; + + int nonosub = REG_NOSUB == 0; + int nonexec = 0; + + unsigned long test = 0; + + static char* filter[] = { "-", 0 }; + + state.NOMATCH.rm_so = state.NOMATCH.rm_eo = -2; + p = unit; + version = (char*)id + 10; + while (p < &unit[sizeof(unit)-1] && (*p = *version++) && !isspace(*p)) + p++; + *p = 0; + while ((p = *++argv) && *p == '-') + for (;;) + { + switch (*++p) + { + case 0: + break; + case 'c': + test |= TEST_CATCH; + continue; + case 'e': + test |= TEST_IGNORE_ERROR; + continue; + case 'h': + case '?': + help(0); + return 2; + case '-': + help(p[1] == 'h'); + return 2; + case 'n': + nonexec = 1; + continue; + case 'o': + test |= TEST_IGNORE_OVER; + continue; + case 'p': + test |= TEST_IGNORE_POSITION; + continue; + case 's': +#ifdef REG_DISCIPLINE + if (!(state.stack = stkalloc(stkstd, 0))) + fprintf(stderr, "%s: out of space [stack]", unit); + state.disc.disc.re_resizef = resizef; + state.disc.disc.re_resizehandle = (void*)stkstd; +#endif + continue; + case 'x': + nonosub = 1; + continue; + case 'v': + test |= TEST_VERBOSE; + continue; + case 'A': + test |= TEST_ACTUAL; + continue; + case 'B': + test |= TEST_BASELINE; + continue; + case 'F': + test |= TEST_FAIL; + continue; + case 'P': + test |= TEST_PASS; + continue; + case 'S': + test |= TEST_SUMMARY; + continue; + default: + fprintf(stderr, "%s: %c: invalid option\n", unit, *p); + return 2; + } + break; + } + if (!*argv) + argv = filter; + locale = 0; + while (state.file = *argv++) + { + if (streq(state.file, "-") || streq(state.file, "/dev/stdin") || streq(state.file, "/dev/fd/0")) + { + state.file = 0; + fp = stdin; + } + else if (!(fp = fopen(state.file, "r"))) + { + fprintf(stderr, "%s: %s: cannot read\n", unit, state.file); + return 2; + } + testno = state.errors = state.ignored = state.lineno = state.passed = + state.signals = state.unspecified = state.warnings = 0; + skip = 0; + level = 1; + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_SUMMARY))) + { + printf("TEST\t%s ", unit); + if (s = state.file) + { + subunit = p = 0; + for (;;) + { + switch (*s++) + { + case 0: + break; + case '/': + subunit = s; + continue; + case '.': + p = s - 1; + continue; + default: + continue; + } + break; + } + if (!subunit) + subunit = state.file; + if (p < subunit) + p = s - 1; + subunitlen = p - subunit; + printf("%-.*s ", subunitlen, subunit); + } + else + subunit = 0; + for (s = version; *s && (*s != ' ' || *(s + 1) != '$'); s++) + putchar(*s); + if (test & TEST_CATCH) + printf(", catch"); + if (test & TEST_IGNORE_ERROR) + printf(", ignore error code mismatches"); + if (test & TEST_IGNORE_POSITION) + printf(", ignore negative position mismatches"); +#ifdef REG_DISCIPLINE + if (state.stack) + printf(", stack"); +#endif + if (test & TEST_VERBOSE) + printf(", verbose"); + printf("\n"); +#ifdef REG_VERSIONID + if (regerror(REG_VERSIONID, NiL, pat, sizeof(pat)) > 0) + s = pat; + else +#endif +#ifdef REG_TEST_VERSION + s = REG_TEST_VERSION; +#else + s = "regex"; +#endif + printf("NOTE\t%s\n", s); + if (elementsof(unsupported) > 1) + { +#if (REG_TEST_DEFAULT & (REG_AUGMENTED|REG_EXTENDED|REG_SHELL)) || !defined(REG_EXTENDED) + i = 0; +#else + i = REG_EXTENDED != 0; +#endif + for (got = 0; i < elementsof(unsupported) - 1; i++) + { + if (!got) + { + got = 1; + printf("NOTE\tunsupported: %s", unsupported[i]); + } + else + printf(",%s", unsupported[i]); + } + if (got) + printf("\n"); + } + } +#ifdef REG_DISCIPLINE + state.disc.disc.re_version = REG_VERSION; + state.disc.disc.re_compf = compf; + state.disc.disc.re_execf = execf; + if (!(state.disc.sp = sfstropen())) + bad("out of space [discipline string stream]\n", NiL, NiL, 0, 0); + preg.re_disc = &state.disc.disc; +#endif + if (test & TEST_CATCH) + { + signal(SIGALRM, gotcha); + signal(SIGBUS, gotcha); + signal(SIGSEGV, gotcha); + } + while (p = getline(fp)) + { + + /* parse: */ + + line = p; + if (*p == ':' && !isspace(*(p + 1))) + { + while (*++p && *p != ':'); + if (!*p++) + { + if (test & TEST_BASELINE) + printf("%s\n", line); + continue; + } + } + while (isspace(*p)) + p++; + if (*p == 0 || *p == '#' || *p == 'T') + { + if (test & TEST_BASELINE) + printf("%s\n", line); + continue; + } + if (*p == ':' || *p == 'N') + { + if (test & TEST_BASELINE) + printf("%s\n", line); + else if (!(test & (TEST_ACTUAL|TEST_FAIL|TEST_PASS|TEST_SUMMARY))) + { + while (*++p && !isspace(*p)); + while (isspace(*p)) + p++; + printf("NOTE %s\n", p); + } + continue; + } + j = 0; + i = 0; + field[i++] = p; + for (;;) + { + switch (*p++) + { + case 0: + p--; + j = 0; + goto checkfield; + case '\t': + *(delim[i] = p - 1) = 0; + j = 1; + checkfield: + s = field[i - 1]; + if (streq(s, "NIL")) + field[i - 1] = 0; + else if (streq(s, "NULL")) + *s = 0; + while (*p == '\t') + { + p++; + j++; + } + tabs[i - 1] = j; + if (!*p) + break; + if (i >= elementsof(field)) + bad("too many fields\n", NiL, NiL, 0, 0); + field[i++] = p; + /*FALLTHROUGH*/ + default: + continue; + } + break; + } + if (!(spec = field[0])) + bad("NIL spec\n", NiL, NiL, 0, 0); + + /* interpret: */ + + cflags = REG_TEST_DEFAULT; + eflags = REG_EXEC_DEFAULT; + test &= TEST_GLOBAL; + state.extracted = 0; + nmatch = 20; + nsub = -1; + for (p = spec; *p; p++) + { + if (isdigit(*p)) + { + nmatch = strtol(p, &p, 10); + if (nmatch >= elementsof(match)) + bad("nmatch must be < 100\n", NiL, NiL, 0, 0); + p--; + continue; + } + switch (*p) + { + case 'A': + test |= TEST_ARE; + continue; + case 'B': + test |= TEST_BRE; + continue; + case 'C': + if (!(test & TEST_QUERY) && !(skip & level)) + bad("locale must be nested\n", NiL, NiL, 0, 0); + test &= ~TEST_QUERY; + if (locale) + bad("locale nesting not supported\n", NiL, NiL, 0, 0); + if (i != 2) + bad("locale field expected\n", NiL, NiL, 0, 0); + if (!(skip & level)) + { +#if defined(LC_COLLATE) && defined(LC_CTYPE) + s = field[1]; + if (!s || streq(s, "POSIX")) + s = "C"; + if ((ans = setlocale(LC_COLLATE, s)) && streq(ans, "POSIX")) + ans = "C"; + if (!ans || !streq(ans, s) && streq(s, "C")) + ans = 0; + else if ((ans = setlocale(LC_CTYPE, s)) && streq(ans, "POSIX")) + ans = "C"; + if (!ans || !streq(ans, s) && streq(s, "C")) + skip = note(level, s, skip, test); + else + { + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_SUMMARY))) + printf("NOTE \"%s\" locale\n", s); + locale = level; + } +#else + skip = note(level, skip, test, "locales not supported"); +#endif + } + cflags = NOTEST; + continue; + case 'E': + test |= TEST_ERE; + continue; + case 'K': + test |= TEST_KRE; + continue; + case 'L': + test |= TEST_LRE; + continue; + case 'S': + test |= TEST_SRE; + continue; + + case 'a': + cflags |= REG_LEFT|REG_RIGHT; + continue; + case 'b': + eflags |= REG_NOTBOL; + continue; + case 'c': + cflags |= REG_COMMENT; + continue; + case 'd': + cflags |= REG_SHELL_DOT; + continue; + case 'e': + eflags |= REG_NOTEOL; + continue; + case 'f': + cflags |= REG_MULTIPLE; + continue; + case 'g': + cflags |= NOTEST; + continue; + case 'h': + cflags |= REG_MULTIREF; + continue; + case 'i': + cflags |= REG_ICASE; + continue; + case 'j': + cflags |= REG_SPAN; + continue; + case 'k': + cflags |= REG_ESCAPE; + continue; + case 'l': + cflags |= REG_LEFT; + continue; + case 'm': + cflags |= REG_MINIMAL; + continue; + case 'n': + cflags |= REG_NEWLINE; + continue; + case 'o': + cflags |= REG_SHELL_GROUP; + continue; + case 'p': + cflags |= REG_SHELL_PATH; + continue; + case 'q': + cflags |= REG_DELIMITED; + continue; + case 'r': + cflags |= REG_RIGHT; + continue; + case 's': + cflags |= REG_SHELL_ESCAPED; + continue; + case 't': + cflags |= REG_MUSTDELIM; + continue; + case 'u': + test |= TEST_UNSPECIFIED; + continue; + case 'v': + cflags |= REG_CLASS_ESCAPE; + continue; + case 'w': + cflags |= REG_NOSUB; + continue; + case 'x': + if (REG_LENIENT) + cflags |= REG_LENIENT; + else + test |= TEST_LENIENT; + continue; + case 'y': + eflags |= REG_LEFT; + continue; + case 'z': + cflags |= REG_NULL; + continue; + + case '$': + test |= TEST_EXPAND; + continue; + + case '/': + test |= TEST_SUB; + continue; + + case '=': + test |= TEST_DECOMP; + continue; + + case '?': + test |= TEST_VERIFY; + test &= ~(TEST_AND|TEST_OR); + state.verify = state.passed; + continue; + case '&': + test |= TEST_VERIFY|TEST_AND; + test &= ~TEST_OR; + continue; + case '|': + test |= TEST_VERIFY|TEST_OR; + test &= ~TEST_AND; + continue; + case ';': + test |= TEST_OR; + test &= ~TEST_AND; + continue; + + case '{': + level <<= 1; + if (skip & (level >> 1)) + { + skip |= level; + cflags = NOTEST; + } + else + { + skip &= ~level; + test |= TEST_QUERY; + } + continue; + case '}': + if (level == 1) + bad("invalid {...} nesting\n", NiL, NiL, 0, 0); + if ((skip & level) && !(skip & (level>>1))) + { + if (!(test & (TEST_BASELINE|TEST_SUMMARY))) + { + if (test & (TEST_ACTUAL|TEST_FAIL)) + printf("}\n"); + else if (!(test & TEST_PASS)) + printf("-%d\n", state.lineno); + } + } +#if defined(LC_COLLATE) && defined(LC_CTYPE) + else if (locale & level) + { + locale = 0; + if (!(skip & level)) + { + s = "C"; + setlocale(LC_COLLATE, s); + setlocale(LC_CTYPE, s); + if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_SUMMARY))) + printf("NOTE \"%s\" locale\n", s); + else if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_PASS)) + printf("}\n"); + } + else if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL)) + printf("}\n"); + } +#endif + level >>= 1; + cflags = NOTEST; + continue; + + default: + bad("bad spec\n", spec, NiL, 0, test); + break; + + } + break; + } + if ((cflags|eflags) == NOTEST || (skip & level) && (test & TEST_BASELINE)) + { + if (test & TEST_BASELINE) + { + while (i > 1) + *delim[--i] = '\t'; + printf("%s\n", line); + } + continue; + } + if (test & TEST_OR) + { + if (!(test & TEST_VERIFY)) + { + test &= ~TEST_OR; + if (state.passed == state.verify && i > 1) + printf("NOTE\t%s\n", field[1]); + continue; + } + else if (state.passed > state.verify) + continue; + } + else if (test & TEST_AND) + { + if (state.passed == state.verify) + continue; + state.passed = state.verify; + } + if (i < ((test & TEST_DECOMP) ? 3 : 4)) + bad("too few fields\n", NiL, NiL, 0, test); + while (i < elementsof(field)) + field[i++] = 0; + if (re = field[1]) + { + if (streq(re, "SAME")) + { + re = ppat; + test |= TEST_SAME; + } + else + { + if (test & TEST_EXPAND) + escape(re); + re = expand(re, patbuf); + strcpy(ppat = pat, re); + } + } + else + ppat = 0; + nstr = -1; + if (s = field[2]) + { + s = expand(s, strbuf); + if (test & TEST_EXPAND) + { + nstr = escape(s); +#if _REG_nexec + if (nstr != strlen(s)) + nexec = nstr; +#endif + } + } + if (!(ans = field[(test & TEST_DECOMP) ? 2 : 3])) + bad("NIL answer\n", NiL, NiL, 0, test); + msg = field[4]; + fflush(stdout); + if (test & TEST_SUB) +#if _REG_subcomp + cflags |= REG_DELIMITED; +#else + continue; +#endif +#if !_REG_decomp + if (test & TEST_DECOMP) + continue; +#endif + + compile: + + if (state.extracted || (skip & level)) + continue; +#if !(REG_TEST_DEFAULT & (REG_AUGMENTED|REG_EXTENDED|REG_SHELL)) +#ifdef REG_EXTENDED + if (REG_EXTENDED != 0 && (test & TEST_BRE)) +#else + if (test & TEST_BRE) +#endif + { + test &= ~TEST_BRE; + flags = cflags; + state.which = "BRE"; + } + else +#endif +#ifdef REG_EXTENDED + if (test & TEST_ERE) + { + test &= ~TEST_ERE; + flags = cflags | REG_EXTENDED; + state.which = "ERE"; + } + else +#endif +#ifdef REG_AUGMENTED + if (test & TEST_ARE) + { + test &= ~TEST_ARE; + flags = cflags | REG_AUGMENTED; + state.which = "ARE"; + } + else +#endif +#ifdef REG_LITERAL + if (test & TEST_LRE) + { + test &= ~TEST_LRE; + flags = cflags | REG_LITERAL; + state.which = "LRE"; + } + else +#endif +#ifdef REG_SHELL + if (test & TEST_SRE) + { + test &= ~TEST_SRE; + flags = cflags | REG_SHELL; + state.which = "SRE"; + } + else +#ifdef REG_AUGMENTED + if (test & TEST_KRE) + { + test &= ~TEST_KRE; + flags = cflags | REG_SHELL | REG_AUGMENTED; + state.which = "KRE"; + } + else +#endif +#endif + { + if (test & (TEST_BASELINE|TEST_PASS|TEST_VERIFY)) + extract(tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test|TEST_OK); + continue; + } + if ((test & (TEST_QUERY|TEST_VERBOSE|TEST_VERIFY)) == TEST_VERBOSE) + { + printf("test %-3d %s ", state.lineno, state.which); + quote(re, -1, test|TEST_DELIMIT); + printf(" "); + quote(s, nstr, test|TEST_DELIMIT); + printf("\n"); + } + + nosub: + fun = "regcomp"; +#if _REG_nexec + if (nstr >= 0 && nstr != strlen(s)) + nexec = nstr; + + else +#endif + nexec = -1; + if (state.extracted || (skip & level)) + continue; + if (!(test & TEST_QUERY)) + testno++; +#ifdef REG_DISCIPLINE + if (state.stack) + stkset(stkstd, state.stack, 0); + flags |= REG_DISCIPLINE; + state.disc.ordinal = 0; + sfstrseek(state.disc.sp, 0, SEEK_SET); +#endif + if (!(test & TEST_CATCH)) + cret = regcomp(&preg, re, flags); + else if (!(cret = setjmp(state.gotcha))) + { + alarm(HUNG); + cret = regcomp(&preg, re, flags); + alarm(0); + } +#if _REG_subcomp + if (!cret && (test & TEST_SUB)) + { + fun = "regsubcomp"; + p = re + preg.re_npat; + if (!(test & TEST_CATCH)) + cret = regsubcomp(&preg, p, NiL, 0, 0); + else if (!(cret = setjmp(state.gotcha))) + { + alarm(HUNG); + cret = regsubcomp(&preg, p, NiL, 0, 0); + alarm(0); + } + if (!cret && *(p += preg.re_npat) && !(preg.re_sub->re_flags & REG_SUB_LAST)) + { + if (catchfree(&preg, flags, tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test)) + continue; + cret = REG_EFLAGS; + } + } +#endif +#if _REG_decomp + if (!cret && (test & TEST_DECOMP)) + { + char buf[128]; + + if ((j = nmatch) > sizeof(buf)) + j = sizeof(buf); + fun = "regdecomp"; + p = re + preg.re_npat; + if (!(test & TEST_CATCH)) + i = regdecomp(&preg, -1, buf, j); + else if (!(cret = setjmp(state.gotcha))) + { + alarm(HUNG); + i = regdecomp(&preg, -1, buf, j); + alarm(0); + } + if (!cret) + { + catchfree(&preg, flags, tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test); + if (i > j) + { + if (i != (strlen(ans) + 1)) + { + report("failed", fun, re, s, nstr, msg, flags, test); + printf(" %d byte buffer supplied, %d byte buffer required\n", j, i); + } + } + else if (strcmp(buf, ans)) + { + report("failed", fun, re, s, nstr, msg, flags, test); + quote(ans, -1, test|TEST_DELIMIT); + printf(" expected, "); + quote(buf, -1, test|TEST_DELIMIT); + printf(" returned\n"); + } + continue; + } + } +#endif + if (!cret) + { + if (!(flags & REG_NOSUB) && nsub < 0 && *ans == '(') + { + for (p = ans; *p; p++) + if (*p == '(') + nsub++; + else if (*p == '{') + nsub--; + if (nsub >= 0) + { + if (test & TEST_IGNORE_OVER) + { + if (nmatch > nsub) + nmatch = nsub + 1; + } + else if (nsub != preg.re_nsub) + { + if (nsub > preg.re_nsub) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, "OK", NiL, 0, 0, skip, level, test|TEST_DELIMIT); + else + { + report("re_nsub incorrect", fun, re, NiL, -1, msg, flags, test); + printf("at least %d expected, %d returned\n", nsub, preg.re_nsub); + state.errors++; + } + } + else + nsub = preg.re_nsub; + } + } + } + if (!(test & (TEST_DECOMP|TEST_SUB)) && *ans && *ans != '(' && !streq(ans, "OK") && !streq(ans, "NOMATCH")) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, "OK", NiL, 0, 0, skip, level, test|TEST_DELIMIT); + else if (!(test & TEST_LENIENT)) + { + report("failed", fun, re, NiL, -1, msg, flags, test); + printf("%s expected, OK returned\n", ans); + } + catchfree(&preg, flags, tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test); + continue; + } + } + else + { + if (test & TEST_LENIENT) + /* we'll let it go this time */; + else if (!*ans || ans[0]=='(' || cret == REG_BADPAT && streq(ans, "NOMATCH")) + { + got = 0; + for (i = 1; i < elementsof(codes); i++) + if (cret==codes[i].code) + got = i; + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, codes[got].name, NiL, 0, 0, skip, level, test|TEST_DELIMIT); + else + { + report("failed", fun, re, NiL, -1, msg, flags, test); + printf("%s returned: ", codes[got].name); + error(&preg, cret); + } + } + else + { + expected = got = 0; + for (i = 1; i < elementsof(codes); i++) + { + if (streq(ans, codes[i].name)) + expected = i; + if (cret==codes[i].code) + got = i; + } + if (!expected) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, codes[got].name, NiL, 0, 0, skip, level, test|TEST_DELIMIT); + else + { + report("failed: invalid error code", NiL, re, NiL, -1, msg, flags, test); + printf("%s expected, %s returned\n", ans, codes[got].name); + } + } + else if (cret != codes[expected].code && cret != REG_BADPAT) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, codes[got].name, NiL, 0, 0, skip, level, test|TEST_DELIMIT); + else if (test & TEST_IGNORE_ERROR) + state.ignored++; + else + { + report("should fail and did", fun, re, NiL, -1, msg, flags, test); + printf("%s expected, %s returned: ", ans, codes[got].name); + state.errors--; + state.warnings++; + error(&preg, cret); + } + } + } + goto compile; + } + +#if _REG_nexec + execute: + if (nexec >= 0) + fun = "regnexec"; + else +#endif + fun = "regexec"; + + for (i = 0; i < elementsof(match); i++) + match[i] = state.NOMATCH; + +#if _REG_nexec + if (nexec >= 0) + { + eret = regnexec(&preg, s, nexec, nmatch, match, eflags); + s[nexec] = 0; + } + else +#endif + { + if (!(test & TEST_CATCH)) + eret = regexec(&preg, s, nmatch, match, eflags); + else if (!(eret = setjmp(state.gotcha))) + { + alarm(HUNG); + eret = regexec(&preg, s, nmatch, match, eflags); + alarm(0); + } + } +#if _REG_subcomp + if ((test & TEST_SUB) && !eret) + { + fun = "regsubexec"; + if (!(test & TEST_CATCH)) + eret = regsubexec(&preg, s, nmatch, match); + else if (!(eret = setjmp(state.gotcha))) + { + alarm(HUNG); + eret = regsubexec(&preg, s, nmatch, match); + alarm(0); + } + } +#endif + if (flags & REG_NOSUB) + { + if (eret) + { + if (eret != REG_NOMATCH || !streq(ans, "NOMATCH")) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, "NOMATCH", NiL, 0, 0, skip, level, test|TEST_DELIMIT); + else + { + report("REG_NOSUB failed", fun, re, s, nstr, msg, flags, test); + error(&preg, eret); + } + } + } + else if (streq(ans, "NOMATCH")) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, NiL, match, nmatch, nsub, skip, level, test|TEST_DELIMIT); + else + { + report("should fail and didn't", fun, re, s, nstr, msg, flags, test); + error(&preg, eret); + } + } + } + else if (eret) + { + if (eret != REG_NOMATCH || !streq(ans, "NOMATCH")) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, "NOMATCH", NiL, 0, nsub, skip, level, test|TEST_DELIMIT); + else + { + report("failed", fun, re, s, nstr, msg, flags, test); + if (eret != REG_NOMATCH) + error(&preg, eret); + else if (*ans) + printf("expected: %s\n", ans); + else + printf("\n"); + } + } + } + else if (streq(ans, "NOMATCH")) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, NiL, match, nmatch, nsub, skip, level, test|TEST_DELIMIT); + else + { + report("should fail and didn't", fun, re, s, nstr, msg, flags, test); + matchprint(match, nmatch, nsub, NiL, test); + } + } +#if _REG_subcomp + else if (test & TEST_SUB) + { + p = preg.re_sub->re_buf; + if (strcmp(p, ans)) + { + report("failed", fun, re, s, nstr, msg, flags, test); + quote(ans, -1, test|TEST_DELIMIT); + printf(" expected, "); + quote(p, -1, test|TEST_DELIMIT); + printf(" returned\n"); + } + } +#endif + else if (!*ans) + { + if (match[0].rm_so != state.NOMATCH.rm_so) + { + if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test); + else + { + report("failed: no match but match array assigned", NiL, re, s, nstr, msg, flags, test); + matchprint(match, nmatch, nsub, NiL, test); + } + } + } + else if (matchcheck(match, nmatch, nsub, ans, re, s, nstr, flags, test)) + { +#if _REG_nexec + if (nexec < 0 && !nonexec) + { + nexec = nstr >= 0 ? nstr : strlen(s); + s[nexec] = '\n'; + testno++; + goto execute; + } +#endif + if (!(test & (TEST_DECOMP|TEST_SUB|TEST_VERIFY)) && !nonosub) + { + if (catchfree(&preg, flags, tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test)) + continue; + flags |= REG_NOSUB; + goto nosub; + } + if (test & (TEST_BASELINE|TEST_PASS|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, NiL, match, nmatch, nsub, skip, level, test|TEST_OK); + } + else if (test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS|TEST_QUERY|TEST_SUMMARY|TEST_VERIFY)) + skip = extract(tabs, line, re, s, ans, msg, NiL, match, nmatch, nsub, skip, level, test|TEST_DELIMIT); + if (catchfree(&preg, flags, tabs, line, re, s, ans, msg, NiL, NiL, 0, 0, skip, level, test)) + continue; + goto compile; + } + if (test & TEST_SUMMARY) + printf("tests=%-4d errors=%-4d warnings=%-2d ignored=%-2d unspecified=%-2d signals=%d\n", testno, state.errors, state.warnings, state.ignored, state.unspecified, state.signals); + else if (!(test & (TEST_ACTUAL|TEST_BASELINE|TEST_FAIL|TEST_PASS))) + { + printf("TEST\t%s", unit); + if (subunit) + printf(" %-.*s", subunitlen, subunit); + printf(", %d test%s", testno, testno == 1 ? "" : "s"); + if (state.ignored) + printf(", %d ignored mismatche%s", state.ignored, state.ignored == 1 ? "" : "s"); + if (state.warnings) + printf(", %d warning%s", state.warnings, state.warnings == 1 ? "" : "s"); + if (state.unspecified) + printf(", %d unspecified difference%s", state.unspecified, state.unspecified == 1 ? "" : "s"); + if (state.signals) + printf(", %d signal%s", state.signals, state.signals == 1 ? "" : "s"); + printf(", %d error%s\n", state.errors, state.errors == 1 ? "" : "s"); + } + if (fp != stdin) + fclose(fp); + } + return 0; +} diff --git a/libgo/go/rpc/client.go b/libgo/go/rpc/client.go index 4acfdf6..3dc6df1 100644 --- a/libgo/go/rpc/client.go +++ b/libgo/go/rpc/client.go @@ -85,7 +85,8 @@ func (client *Client) send(c *Call) { client.request.Seq = c.seq client.request.ServiceMethod = c.ServiceMethod if err := client.codec.WriteRequest(&client.request, c.Args); err != nil { - panic("rpc: client encode error: " + err.String()) + c.Error = err + c.done() } } @@ -104,7 +105,7 @@ func (client *Client) input() { seq := response.Seq client.mutex.Lock() c := client.pending[seq] - client.pending[seq] = c, false + delete(client.pending, seq) client.mutex.Unlock() if response.Error == "" { @@ -251,10 +252,10 @@ func (client *Client) Close() os.Error { // the same Call object. If done is nil, Go will allocate a new channel. // If non-nil, done must be buffered or Go will deliberately crash. func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call { - c := new(Call) - c.ServiceMethod = serviceMethod - c.Args = args - c.Reply = reply + call := new(Call) + call.ServiceMethod = serviceMethod + call.Args = args + call.Reply = reply if done == nil { done = make(chan *Call, 10) // buffered. } else { @@ -266,14 +267,14 @@ func (client *Client) Go(serviceMethod string, args interface{}, reply interface log.Panic("rpc: done channel is unbuffered") } } - c.Done = done + call.Done = done if client.shutdown { - c.Error = ErrShutdown - c.done() - return c + call.Error = ErrShutdown + call.done() + return call } - client.send(c) - return c + client.send(call) + return call } // Call invokes the named function, waits for it to complete, and returns its error status. diff --git a/libgo/go/rpc/jsonrpc/client.go b/libgo/go/rpc/jsonrpc/client.go index 577d0ce..17e9b93 100644 --- a/libgo/go/rpc/jsonrpc/client.go +++ b/libgo/go/rpc/jsonrpc/client.go @@ -79,7 +79,7 @@ func (c *clientCodec) ReadResponseHeader(r *rpc.Response) os.Error { c.mutex.Lock() r.ServiceMethod = c.pending[c.resp.Id] - c.pending[c.resp.Id] = "", false + delete(c.pending, c.resp.Id) c.mutex.Unlock() r.Error = "" diff --git a/libgo/go/rpc/jsonrpc/server.go b/libgo/go/rpc/jsonrpc/server.go index 9801fdf..61b5abf 100644 --- a/libgo/go/rpc/jsonrpc/server.go +++ b/libgo/go/rpc/jsonrpc/server.go @@ -107,7 +107,7 @@ func (c *serverCodec) WriteResponse(r *rpc.Response, x interface{}) os.Error { c.mutex.Unlock() return os.NewError("invalid sequence number in response") } - c.pending[r.Seq] = nil, false + delete(c.pending, r.Seq) c.mutex.Unlock() if b == nil { diff --git a/libgo/go/rpc/server.go b/libgo/go/rpc/server.go index 7450744..f037100 100644 --- a/libgo/go/rpc/server.go +++ b/libgo/go/rpc/server.go @@ -97,7 +97,7 @@ if err != nil { log.Fatal("arith error:", err) } - fmt.Printf("Arith: %d*%d=%d", args.A, args.B, *reply) + fmt.Printf("Arith: %d*%d=%d", args.A, args.B, reply) or diff --git a/libgo/go/rpc/server_test.go b/libgo/go/rpc/server_test.go index e7bbfbe..029741b 100644 --- a/libgo/go/rpc/server_test.go +++ b/libgo/go/rpc/server_test.go @@ -14,6 +14,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -466,6 +467,32 @@ func TestCountMallocsOverHTTP(t *testing.T) { fmt.Printf("mallocs per HTTP rpc round trip: %d\n", countMallocs(dialHTTP, t)) } +type writeCrasher struct{} + +func (writeCrasher) Close() os.Error { + return nil +} + +func (writeCrasher) Read(p []byte) (int, os.Error) { + return 0, os.EOF +} + +func (writeCrasher) Write(p []byte) (int, os.Error) { + return 0, os.NewError("fake write failure") +} + +func TestClientWriteError(t *testing.T) { + c := NewClient(writeCrasher{}) + res := false + err := c.Call("foo", 1, &res) + if err == nil { + t.Fatal("expected error") + } + if err.String() != "fake write failure" { + t.Error("unexpected value of error:", err) + } +} + func benchmarkEndToEnd(dial func() (*Client, os.Error), b *testing.B) { b.StopTimer() once.Do(startServer) @@ -477,19 +504,79 @@ func benchmarkEndToEnd(dial func() (*Client, os.Error), b *testing.B) { // Synchronous calls args := &Args{7, 8} - reply := new(Reply) + procs := runtime.GOMAXPROCS(-1) + N := int32(b.N) + var wg sync.WaitGroup + wg.Add(procs) b.StartTimer() - for i := 0; i < b.N; i++ { - err = client.Call("Arith.Add", args, reply) - if err != nil { - fmt.Printf("Add: expected no error but got string %q", err.String()) - break - } - if reply.C != args.A+args.B { - fmt.Printf("Add: expected %d got %d", reply.C, args.A+args.B) - break - } + + for p := 0; p < procs; p++ { + go func() { + reply := new(Reply) + for atomic.AddInt32(&N, -1) >= 0 { + err = client.Call("Arith.Add", args, reply) + if err != nil { + fmt.Printf("Add: expected no error but got string %q", err.String()) + panic("rpc error") + } + if reply.C != args.A+args.B { + fmt.Printf("Add: expected %d got %d", reply.C, args.A+args.B) + panic("rpc error") + } + } + wg.Done() + }() + } + wg.Wait() +} + +func benchmarkEndToEndAsync(dial func() (*Client, os.Error), b *testing.B) { + const MaxConcurrentCalls = 100 + b.StopTimer() + once.Do(startServer) + client, err := dial() + if err != nil { + fmt.Println("error dialing", err) + return } + + // Asynchronous calls + args := &Args{7, 8} + procs := 4 * runtime.GOMAXPROCS(-1) + send := int32(b.N) + recv := int32(b.N) + var wg sync.WaitGroup + wg.Add(procs) + gate := make(chan bool, MaxConcurrentCalls) + res := make(chan *Call, MaxConcurrentCalls) + b.StartTimer() + + for p := 0; p < procs; p++ { + go func() { + for atomic.AddInt32(&send, -1) >= 0 { + gate <- true + reply := new(Reply) + client.Go("Arith.Add", args, reply, res) + } + }() + go func() { + for call := range res { + a := call.Args.(*Args).A + b := call.Args.(*Args).B + c := call.Reply.(*Reply).C + if a+b != c { + fmt.Printf("Add: expected %d got %d", a+b, c) + panic("incorrect reply") + } + <-gate + if atomic.AddInt32(&recv, -1) == 0 { + close(res) + } + } + wg.Done() + }() + } + wg.Wait() } func BenchmarkEndToEnd(b *testing.B) { @@ -499,3 +586,11 @@ func BenchmarkEndToEnd(b *testing.B) { func BenchmarkEndToEndHTTP(b *testing.B) { benchmarkEndToEnd(dialHTTP, b) } + +func BenchmarkEndToEndAsync(b *testing.B) { + benchmarkEndToEndAsync(dialDirect, b) +} + +func BenchmarkEndToEndAsyncHTTP(b *testing.B) { + benchmarkEndToEndAsync(dialHTTP, b) +} diff --git a/libgo/go/runtime/chan_test.go b/libgo/go/runtime/chan_test.go index 46ddfd7..7cea906 100644 --- a/libgo/go/runtime/chan_test.go +++ b/libgo/go/runtime/chan_test.go @@ -59,6 +59,57 @@ func TestPseudoRandomSend(t *testing.T) { t.Errorf("Want pseudo random, got %d zeros and %d ones", n0, n1) } +func TestMultiConsumer(t *testing.T) { + const nwork = 23 + const niter = 271828 + + pn := []int{2, 3, 7, 11, 13, 17, 19, 23, 27, 31} + + q := make(chan int, nwork*3) + r := make(chan int, nwork*3) + + // workers + var wg sync.WaitGroup + for i := 0; i < nwork; i++ { + wg.Add(1) + go func(w int) { + for v := range q { + // mess with the fifo-ish nature of range + if pn[w%len(pn)] == v { + runtime.Gosched() + } + r <- v + } + wg.Done() + }(i) + } + + // feeder & closer + expect := 0 + go func() { + for i := 0; i < niter; i++ { + v := pn[i%len(pn)] + expect += v + q <- v + } + close(q) // no more work + wg.Wait() // workers done + close(r) // ... so there can be no more results + }() + + // consume & check + n := 0 + s := 0 + for v := range r { + n++ + s += v + } + if n != niter || s != expect { + t.Errorf("Expected sum %d (got %d) from %d iter (saw %d)", + expect, s, niter, n) + } +} + func BenchmarkSelectUncontended(b *testing.B) { const CallsPerSched = 1000 procs := runtime.GOMAXPROCS(-1) diff --git a/libgo/go/runtime/extern.go b/libgo/go/runtime/extern.go index 9da3423..7c986da 100644 --- a/libgo/go/runtime/extern.go +++ b/libgo/go/runtime/extern.go @@ -131,8 +131,8 @@ func Semrelease(s *uint32) // The argument x must be a pointer to an object allocated by // calling new or by taking the address of a composite literal. // The argument f must be a function that takes a single argument -// of x's type and returns no arguments. If either of these is not -// true, SetFinalizer aborts the program. +// of x's type and can have arbitrary ignored return values. +// If either of these is not true, SetFinalizer aborts the program. // // Finalizers are run in dependency order: if A points at B, both have // finalizers, and they are otherwise unreachable, only the finalizer @@ -156,9 +156,6 @@ func Semrelease(s *uint32) // A single goroutine runs all finalizers for a program, sequentially. // If a finalizer must run for a long time, it should do so by starting // a new goroutine. -// -// TODO(rsc): allow f to have (ignored) return values -// func SetFinalizer(x, f interface{}) func getgoroot() string diff --git a/libgo/go/runtime/gc_test.go b/libgo/go/runtime/gc_test.go new file mode 100644 index 0000000..fad60a3 --- /dev/null +++ b/libgo/go/runtime/gc_test.go @@ -0,0 +1,24 @@ +package runtime_test + +import ( + "runtime" + "testing" +) + +func TestGcSys(t *testing.T) { + for i := 0; i < 1000000; i++ { + workthegc() + } + + // Should only be using a few MB. + runtime.UpdateMemStats() + sys := runtime.MemStats.Sys + t.Logf("using %d MB", sys>>20) + if sys > 10e6 { + t.Fatalf("using too much memory: %d MB", sys>>20) + } +} + +func workthegc() []byte { + return make([]byte, 1029) +} diff --git a/libgo/go/runtime/mfinal_test.go b/libgo/go/runtime/mfinal_test.go new file mode 100644 index 0000000..de63271 --- /dev/null +++ b/libgo/go/runtime/mfinal_test.go @@ -0,0 +1,64 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "runtime" + "sync" + "sync/atomic" + "testing" +) + +func fin(v *int) { +} + +func BenchmarkFinalizer(b *testing.B) { + const CallsPerSched = 1000 + procs := runtime.GOMAXPROCS(-1) + N := int32(b.N / CallsPerSched) + var wg sync.WaitGroup + wg.Add(procs) + for p := 0; p < procs; p++ { + go func() { + var data [CallsPerSched]*int + for i := 0; i < CallsPerSched; i++ { + data[i] = new(int) + } + for atomic.AddInt32(&N, -1) >= 0 { + runtime.Gosched() + for i := 0; i < CallsPerSched; i++ { + runtime.SetFinalizer(data[i], fin) + } + for i := 0; i < CallsPerSched; i++ { + runtime.SetFinalizer(data[i], nil) + } + } + wg.Done() + }() + } + wg.Wait() +} + +func BenchmarkFinalizerRun(b *testing.B) { + const CallsPerSched = 1000 + procs := runtime.GOMAXPROCS(-1) + N := int32(b.N / CallsPerSched) + var wg sync.WaitGroup + wg.Add(procs) + for p := 0; p < procs; p++ { + go func() { + for atomic.AddInt32(&N, -1) >= 0 { + runtime.Gosched() + for i := 0; i < CallsPerSched; i++ { + v := new(int) + runtime.SetFinalizer(v, fin) + } + runtime.GC() + } + wg.Done() + }() + } + wg.Wait() +} diff --git a/libgo/go/runtime/pprof/pprof_test.go b/libgo/go/runtime/pprof/pprof_test.go index 4486d55..5f128c0 100644 --- a/libgo/go/runtime/pprof/pprof_test.go +++ b/libgo/go/runtime/pprof/pprof_test.go @@ -22,9 +22,6 @@ func TestCPUProfile(t *testing.T) { case "plan9": // unimplemented return - case "windows": - // unimplemented - return } buf := make([]byte, 100000) diff --git a/libgo/go/scanner/scanner.go b/libgo/go/scanner/scanner.go index 8fbcb9c..29e5f8c 100644 --- a/libgo/go/scanner/scanner.go +++ b/libgo/go/scanner/scanner.go @@ -164,9 +164,12 @@ type Scanner struct { // for values ch > ' '). The field may be changed at any time. Whitespace uint64 - // Current token position. The Offset, Line, and Column fields - // are set by Scan(); the Filename field is left untouched by the - // Scanner. + // Start position of most recently scanned token; set by Scan. + // Calling Init or Next invalidates the position (Line == 0). + // The Filename field is always left untouched by the Scanner. + // If an error is reported (via Error) and Position is invalid, + // the scanner is not inside a token. Call Pos to obtain an error + // position in that case. Position } @@ -201,6 +204,7 @@ func (s *Scanner) Init(src io.Reader) *Scanner { s.ErrorCount = 0 s.Mode = GoTokens s.Whitespace = GoWhitespace + s.Line = 0 // invalidate token position return s } @@ -302,6 +306,7 @@ func (s *Scanner) next() int { // get the current position. func (s *Scanner) Next() int { s.tokPos = -1 // don't collect token text + s.Line = 0 // invalidate token position ch := s.Peek() s.ch = s.next() return ch @@ -323,7 +328,11 @@ func (s *Scanner) error(msg string) { s.Error(s, msg) return } - fmt.Fprintf(os.Stderr, "%s: %s\n", s.Position, msg) + pos := s.Position + if !pos.IsValid() { + pos = s.Pos() + } + fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg) } func (s *Scanner) scanIdentifier() int { @@ -520,6 +529,7 @@ func (s *Scanner) Scan() int { // reset token text position s.tokPos = -1 + s.Line = 0 redo: // skip white space diff --git a/libgo/go/scanner/scanner_test.go b/libgo/go/scanner/scanner_test.go index 8403d61..bbbba12 100644 --- a/libgo/go/scanner/scanner_test.go +++ b/libgo/go/scanner/scanner_test.go @@ -100,11 +100,8 @@ var tokenList = []token{ {Ident, "_abc_123_"}, {Ident, "_äöü"}, {Ident, "_本"}, - // TODO for unknown reasons these fail when checking the literals - /* - token{Ident, "äöü"}, - token{Ident, "本"}, - */ + {Ident, "äöü"}, + {Ident, "本"}, {Ident, "a۰۱۸"}, {Ident, "foo६४"}, {Ident, "bar９８７６"}, @@ -365,14 +362,14 @@ func TestScanNext(t *testing.T) { checkTok(t, s, 1, s.Scan(), Ident, "if") checkTok(t, s, 1, s.Scan(), Ident, "a") checkTok(t, s, 1, s.Scan(), '=', "=") - checkTok(t, s, 1, s.Next(), '=', "") - checkTok(t, s, 1, s.Next(), ' ', "") - checkTok(t, s, 1, s.Next(), 'b', "") + checkTok(t, s, 0, s.Next(), '=', "") + checkTok(t, s, 0, s.Next(), ' ', "") + checkTok(t, s, 0, s.Next(), 'b', "") checkTok(t, s, 1, s.Scan(), Ident, "cd") checkTok(t, s, 1, s.Scan(), '{', "{") checkTok(t, s, 2, s.Scan(), Ident, "a") checkTok(t, s, 2, s.Scan(), '+', "+") - checkTok(t, s, 2, s.Next(), '=', "") + checkTok(t, s, 0, s.Next(), '=', "") checkTok(t, s, 2, s.Scan(), Ident, "c") checkTok(t, s, 3, s.Scan(), '}', "}") checkTok(t, s, 3, s.Scan(), -1, "") diff --git a/libgo/go/sort/example_test.go b/libgo/go/sort/example_test.go new file mode 100644 index 0000000..2f5ee90 --- /dev/null +++ b/libgo/go/sort/example_test.go @@ -0,0 +1,17 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sort_test + +import ( + "fmt" + "sort" +) + +// [1 2 3 4 5 6] +func ExampleInts() { + s := []int{5, 2, 6, 3, 1, 4} + sort.Ints(s) + fmt.Println(s) +} diff --git a/libgo/go/sort/export_test.go b/libgo/go/sort/export_test.go new file mode 100644 index 0000000..b6e30ce --- /dev/null +++ b/libgo/go/sort/export_test.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sort + +func Heapsort(data Interface) { + heapSort(data, 0, data.Len()) +} diff --git a/libgo/go/sort/sort.go b/libgo/go/sort/sort.go index 0a4a437..83ee170 100644 --- a/libgo/go/sort/sort.go +++ b/libgo/go/sort/sort.go @@ -37,10 +37,47 @@ func insertionSort(data Interface, a, b int) { } } +// siftDown implements the heap property on data[lo, hi). +// first is an offset into the array where the root of the heap lies. +func siftDown(data Interface, lo, hi, first int) { + root := lo + for { + child := 2*root + 1 + if child >= hi { + break + } + if child+1 < hi && data.Less(first+child, first+child+1) { + child++ + } + if !data.Less(first+root, first+child) { + return + } + data.Swap(first+root, first+child) + root = child + } +} + +func heapSort(data Interface, a, b int) { + first := a + lo := 0 + hi := b - a + + // Build heap with greatest element at top. + for i := (hi - 1) / 2; i >= 0; i-- { + siftDown(data, i, hi, first) + } + + // Pop elements, largest first, into end of data. + for i := hi - 1; i >= 0; i-- { + data.Swap(first, first+i) + siftDown(data, lo, i, first) + } +} + // Quicksort, following Bentley and McIlroy, // ``Engineering a Sort Function,'' SP&E November 1993. -// Move the median of the three values data[a], data[b], data[c] into data[a]. +// medianOfThree moves the median of the three values data[a], data[b], data[c] into data[a]. func medianOfThree(data Interface, a, b, c int) { m0 := b m1 := a @@ -123,16 +160,21 @@ func doPivot(data Interface, lo, hi int) (midlo, midhi int) { return lo + b - a, hi - (d - c) } -func quickSort(data Interface, a, b int) { +func quickSort(data Interface, a, b, maxDepth int) { for b-a > 7 { + if maxDepth == 0 { + heapSort(data, a, b) + return + } + maxDepth-- mlo, mhi := doPivot(data, a, b) // Avoiding recursion on the larger subproblem guarantees // a stack depth of at most lg(b-a). if mlo-a < b-mhi { - quickSort(data, a, mlo) + quickSort(data, a, mlo, maxDepth) a = mhi // i.e., quickSort(data, mhi, b) } else { - quickSort(data, mhi, b) + quickSort(data, mhi, b, maxDepth) b = mlo // i.e., quickSort(data, a, mlo) } } @@ -141,7 +183,16 @@ func quickSort(data Interface, a, b int) { } } -func Sort(data Interface) { quickSort(data, 0, data.Len()) } +func Sort(data Interface) { + // Switch to heapsort if depth of 2*ceil(lg(n)) is reached. + n := data.Len() + maxDepth := 0 + for 1<<uint(maxDepth) < n { + maxDepth++ + } + maxDepth *= 2 + quickSort(data, 0, data.Len(), maxDepth) +} func IsSorted(data Interface) bool { n := data.Len() diff --git a/libgo/go/sort/sort_test.go b/libgo/go/sort/sort_test.go index 5007a92..a564015 100644 --- a/libgo/go/sort/sort_test.go +++ b/libgo/go/sort/sort_test.go @@ -169,6 +169,13 @@ func (d *testingData) Swap(i, j int) { d.data[i], d.data[j] = d.data[j], d.data[i] } +func min(a, b int) int { + if a < b { + return a + } + return b +} + func lg(n int) int { i := 0 for 1<<uint(i) < n { @@ -177,7 +184,7 @@ func lg(n int) int { return i } -func TestBentleyMcIlroy(t *testing.T) { +func testBentleyMcIlroy(t *testing.T, sort func(Interface)) { sizes := []int{100, 1023, 1024, 1025} if testing.Short() { sizes = []int{100, 127, 128, 129} @@ -253,7 +260,7 @@ func TestBentleyMcIlroy(t *testing.T) { desc := fmt.Sprintf("n=%d m=%d dist=%s mode=%s", n, m, dists[dist], modes[mode]) d := &testingData{desc, t, mdata[0:n], n * lg(n) * 12 / 10, 0} - Sort(d) + sort(d) // If we were testing C qsort, we'd have to make a copy // of the slice and sort it ourselves and then compare @@ -274,9 +281,58 @@ func TestBentleyMcIlroy(t *testing.T) { } } -func min(a, b int) int { - if a < b { - return a +func TestSortBM(t *testing.T) { + testBentleyMcIlroy(t, Sort) +} + +func TestHeapsortBM(t *testing.T) { + testBentleyMcIlroy(t, Heapsort) +} + +// This is based on the "antiquicksort" implementation by M. Douglas McIlroy. +// See http://www.cs.dartmouth.edu/~doug/mdmspe.pdf for more info. +type adversaryTestingData struct { + data []int + keys map[int]int + candidate int +} + +func (d *adversaryTestingData) Len() int { return len(d.data) } + +func (d *adversaryTestingData) Less(i, j int) bool { + if _, present := d.keys[i]; !present { + if _, present := d.keys[j]; !present { + if i == d.candidate { + d.keys[i] = len(d.keys) + } else { + d.keys[j] = len(d.keys) + } + } } - return b + + if _, present := d.keys[i]; !present { + d.candidate = i + return false + } + if _, present := d.keys[j]; !present { + d.candidate = j + return true + } + + return d.keys[i] >= d.keys[j] +} + +func (d *adversaryTestingData) Swap(i, j int) { + d.data[i], d.data[j] = d.data[j], d.data[i] +} + +func TestAdversary(t *testing.T) { + const size = 100 + data := make([]int, size) + for i := 0; i < size; i++ { + data[i] = i + } + + d := &adversaryTestingData{data, make(map[int]int), 0} + Sort(d) // This should degenerate to heapsort. } diff --git a/libgo/go/strconv/atof.go b/libgo/go/strconv/atof.go index 38b3805..86c56f7 100644 --- a/libgo/go/strconv/atof.go +++ b/libgo/go/strconv/atof.go @@ -56,8 +56,9 @@ func special(s string) (f float64, ok bool) { } // TODO(rsc): Better truncation handling. -func stringToDecimal(s string) (neg bool, d *decimal, trunc bool, ok bool) { +func (b *decimal) set(s string) (ok bool) { i := 0 + b.neg = false // optional sign if i >= len(s) { @@ -67,12 +68,11 @@ func stringToDecimal(s string) (neg bool, d *decimal, trunc bool, ok bool) { case s[i] == '+': i++ case s[i] == '-': - neg = true + b.neg = true i++ } // digits - b := new(decimal) sawdot := false sawdigits := false for ; i < len(s); i++ { @@ -137,7 +137,6 @@ func stringToDecimal(s string) (neg bool, d *decimal, trunc bool, ok bool) { return } - d = b ok = true return } @@ -145,7 +144,7 @@ func stringToDecimal(s string) (neg bool, d *decimal, trunc bool, ok bool) { // decimal power of ten to binary power of two. var powtab = []int{1, 3, 6, 9, 13, 16, 19, 23, 26} -func decimalToFloatBits(neg bool, d *decimal, trunc bool, flt *floatInfo) (b uint64, overflow bool) { +func (d *decimal) floatBits(flt *floatInfo) (b uint64, overflow bool) { var exp int var mant uint64 @@ -209,7 +208,8 @@ func decimalToFloatBits(neg bool, d *decimal, trunc bool, flt *floatInfo) (b uin } // Extract 1+flt.mantbits bits. - mant = d.Shift(int(1 + flt.mantbits)).RoundedInteger() + d.Shift(int(1 + flt.mantbits)) + mant = d.RoundedInteger() // Rounding might have added a bit; shift down. if mant == 2<<flt.mantbits { @@ -236,7 +236,7 @@ out: // Assemble bits. bits := mant & (uint64(1)<<flt.mantbits - 1) bits |= uint64((exp-flt.bias)&(1<<flt.expbits-1)) << flt.mantbits - if neg { + if d.neg { bits |= 1 << flt.mantbits << flt.expbits } return bits, overflow @@ -244,24 +244,24 @@ out: // Compute exact floating-point integer from d's digits. // Caller is responsible for avoiding overflow. -func decimalAtof64Int(neg bool, d *decimal) float64 { +func (d *decimal) atof64int() float64 { f := 0.0 for i := 0; i < d.nd; i++ { f = f*10 + float64(d.d[i]-'0') } - if neg { - f *= -1 // BUG work around 6g f = -f. + if d.neg { + f = -f } return f } -func decimalAtof32Int(neg bool, d *decimal) float32 { +func (d *decimal) atof32int() float32 { f := float32(0) for i := 0; i < d.nd; i++ { f = f*10 + float32(d.d[i]-'0') } - if neg { - f *= -1 // BUG work around 6g f = -f. + if d.neg { + f = -f } return f } @@ -281,7 +281,7 @@ var float32pow10 = []float32{1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1 // value is exact integer * exact power of ten // value is exact integer / exact power of ten // These all produce potentially inexact but correctly rounded answers. -func decimalAtof64(neg bool, d *decimal, trunc bool) (f float64, ok bool) { +func (d *decimal) atof64() (f float64, ok bool) { // Exact integers are <= 10^15. // Exact powers of ten are <= 10^22. if d.nd > 15 { @@ -289,11 +289,11 @@ func decimalAtof64(neg bool, d *decimal, trunc bool) (f float64, ok bool) { } switch { case d.dp == d.nd: // int - f := decimalAtof64Int(neg, d) + f := d.atof64int() return f, true case d.dp > d.nd && d.dp <= 15+22: // int * 10^k - f := decimalAtof64Int(neg, d) + f := d.atof64int() k := d.dp - d.nd // If exponent is big but number of digits is not, // can move a few zeros into the integer part. @@ -304,7 +304,7 @@ func decimalAtof64(neg bool, d *decimal, trunc bool) (f float64, ok bool) { return f * float64pow10[k], true case d.dp < d.nd && d.nd-d.dp <= 22: // int / 10^k - f := decimalAtof64Int(neg, d) + f := d.atof64int() return f / float64pow10[d.nd-d.dp], true } return @@ -312,7 +312,7 @@ func decimalAtof64(neg bool, d *decimal, trunc bool) (f float64, ok bool) { // If possible to convert decimal d to 32-bit float f exactly, // entirely in floating-point math, do so, avoiding the machinery above. -func decimalAtof32(neg bool, d *decimal, trunc bool) (f float32, ok bool) { +func (d *decimal) atof32() (f float32, ok bool) { // Exact integers are <= 10^7. // Exact powers of ten are <= 10^10. if d.nd > 7 { @@ -320,11 +320,11 @@ func decimalAtof32(neg bool, d *decimal, trunc bool) (f float32, ok bool) { } switch { case d.dp == d.nd: // int - f := decimalAtof32Int(neg, d) + f := d.atof32int() return f, true case d.dp > d.nd && d.dp <= 7+10: // int * 10^k - f := decimalAtof32Int(neg, d) + f := d.atof32int() k := d.dp - d.nd // If exponent is big but number of digits is not, // can move a few zeros into the integer part. @@ -335,7 +335,7 @@ func decimalAtof32(neg bool, d *decimal, trunc bool) (f float32, ok bool) { return f * float32pow10[k], true case d.dp < d.nd && d.nd-d.dp <= 10: // int / 10^k - f := decimalAtof32Int(neg, d) + f := d.atof32int() return f / float32pow10[d.nd-d.dp], true } return @@ -360,16 +360,16 @@ func Atof32(s string) (f float32, err os.Error) { return float32(val), nil } - neg, d, trunc, ok := stringToDecimal(s) - if !ok { + var d decimal + if !d.set(s) { return 0, &NumError{s, os.EINVAL} } if optimize { - if f, ok := decimalAtof32(neg, d, trunc); ok { + if f, ok := d.atof32(); ok { return f, nil } } - b, ovf := decimalToFloatBits(neg, d, trunc, &float32info) + b, ovf := d.floatBits(&float32info) f = math.Float32frombits(uint32(b)) if ovf { err = &NumError{s, os.ERANGE} @@ -385,16 +385,16 @@ func Atof64(s string) (f float64, err os.Error) { return val, nil } - neg, d, trunc, ok := stringToDecimal(s) - if !ok { + var d decimal + if !d.set(s) { return 0, &NumError{s, os.EINVAL} } if optimize { - if f, ok := decimalAtof64(neg, d, trunc); ok { + if f, ok := d.atof64(); ok { return f, nil } } - b, ovf := decimalToFloatBits(neg, d, trunc, &float64info) + b, ovf := d.floatBits(&float64info) f = math.Float64frombits(b) if ovf { err = &NumError{s, os.ERANGE} diff --git a/libgo/go/strconv/atof_test.go b/libgo/go/strconv/atof_test.go index 0fdd0ea..23aafc1 100644 --- a/libgo/go/strconv/atof_test.go +++ b/libgo/go/strconv/atof_test.go @@ -34,6 +34,7 @@ var atoftests = []atofTest{ {"100000000000000016777215", "1.0000000000000001e+23", nil}, {"100000000000000016777216", "1.0000000000000003e+23", nil}, {"-1", "-1", nil}, + {"-0.1", "-0.1", nil}, {"-0", "-0", nil}, {"1e-20", "1e-20", nil}, {"625e-3", "0.625", nil}, diff --git a/libgo/go/strconv/decimal.go b/libgo/go/strconv/decimal.go index 783065b..f572ea4 100644 --- a/libgo/go/strconv/decimal.go +++ b/libgo/go/strconv/decimal.go @@ -14,9 +14,10 @@ package strconv type decimal struct { // TODO(rsc): Can make d[] a bit smaller and add // truncated bool; - d [2000]byte // digits - nd int // number of digits used - dp int // decimal point + d [2000]byte // digits + nd int // number of digits used + dp int // decimal point + neg bool } func (a *decimal) String() string { @@ -266,8 +267,7 @@ func leftShift(a *decimal, k uint) { } // Binary shift left (k > 0) or right (k < 0). -// Returns receiver for convenience. -func (a *decimal) Shift(k int) *decimal { +func (a *decimal) Shift(k int) { switch { case a.nd == 0: // nothing to do: a == 0 @@ -284,7 +284,6 @@ func (a *decimal) Shift(k int) *decimal { } rightShift(a, uint(-k)) } - return a } // If we chop a at nd digits, should we round up? diff --git a/libgo/go/strconv/decimal_test.go b/libgo/go/strconv/decimal_test.go index 9b79035..deb2e02 100644 --- a/libgo/go/strconv/decimal_test.go +++ b/libgo/go/strconv/decimal_test.go @@ -32,7 +32,9 @@ var shifttests = []shiftTest{ func TestDecimalShift(t *testing.T) { for i := 0; i < len(shifttests); i++ { test := &shifttests[i] - s := NewDecimal(test.i).Shift(test.shift).String() + d := NewDecimal(test.i) + d.Shift(test.shift) + s := d.String() if s != test.out { t.Errorf("Decimal %v << %v = %v, want %v", test.i, test.shift, s, test.out) @@ -108,7 +110,9 @@ var roundinttests = []roundIntTest{ func TestDecimalRoundedInteger(t *testing.T) { for i := 0; i < len(roundinttests); i++ { test := roundinttests[i] - int := NewDecimal(test.i).Shift(test.shift).RoundedInteger() + d := NewDecimal(test.i) + d.Shift(test.shift) + int := d.RoundedInteger() if int != test.int { t.Errorf("Decimal %v >> %v RoundedInteger = %v, want %v", test.i, test.shift, int, test.int) diff --git a/libgo/go/strconv/ftoa.go b/libgo/go/strconv/ftoa.go index b6049c5..07fe806 100644 --- a/libgo/go/strconv/ftoa.go +++ b/libgo/go/strconv/ftoa.go @@ -98,7 +98,8 @@ func genericFtoa(bits uint64, fmt byte, prec int, flt *floatInfo) string { // The shift is exp - flt.mantbits because mant is a 1-bit integer // followed by a flt.mantbits fraction, and we are treating it as // a 1+flt.mantbits-bit integer. - d := newDecimal(mant).Shift(exp - int(flt.mantbits)) + d := newDecimal(mant) + d.Shift(exp - int(flt.mantbits)) // Round appropriately. // Negative precision means "only as much as needed to be exact." @@ -183,7 +184,8 @@ func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { // d = mant << (exp - mantbits) // Next highest floating point number is mant+1 << exp-mantbits. // Our upper bound is halfway inbetween, mant*2+1 << exp-mantbits-1. - upper := newDecimal(mant*2 + 1).Shift(exp - int(flt.mantbits) - 1) + upper := newDecimal(mant*2 + 1) + upper.Shift(exp - int(flt.mantbits) - 1) // d = mant << (exp - mantbits) // Next lowest floating point number is mant-1 << exp-mantbits, @@ -201,7 +203,8 @@ func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { mantlo = mant*2 - 1 explo = exp - 1 } - lower := newDecimal(mantlo*2 + 1).Shift(explo - int(flt.mantbits) - 1) + lower := newDecimal(mantlo*2 + 1) + lower.Shift(explo - int(flt.mantbits) - 1) // The upper and lower bounds are possible outputs only if // the original mantissa is even, so that IEEE round-to-even diff --git a/libgo/go/strconv/quote.go b/libgo/go/strconv/quote.go index 05e49d3..bbb9783 100644 --- a/libgo/go/strconv/quote.go +++ b/libgo/go/strconv/quote.go @@ -288,6 +288,22 @@ func Unquote(s string) (t string, err os.Error) { if quote != '"' && quote != '\'' { return "", os.EINVAL } + if strings.Index(s, "\n") >= 0 { + return "", os.EINVAL + } + + // Is it trivial? Avoid allocation. + if strings.Index(s, `\`) < 0 && strings.IndexRune(s, int(quote)) < 0 { + switch quote { + case '"': + return s, nil + case '\'': + r, size := utf8.DecodeRuneInString(s) + if size == len(s) && (r != utf8.RuneError || size != 1) { + return s, nil + } + } + } var buf bytes.Buffer for len(s) > 0 { diff --git a/libgo/go/strconv/quote_test.go b/libgo/go/strconv/quote_test.go index 4d615db..0311f77 100644 --- a/libgo/go/strconv/quote_test.go +++ b/libgo/go/strconv/quote_test.go @@ -168,6 +168,7 @@ var unquotetests = []unQuoteTest{ {"`\\xFF`", `\xFF`}, {"`\\377`", `\377`}, {"`\\`", `\`}, + {"`\n`", "\n"}, {"` `", ` `}, {"` `", ` `}, } @@ -189,6 +190,9 @@ var misquoted = []string{ "`\"", `"\'"`, `'\"'`, + "\"\n\"", + "\"\\n\n\"", + "'\n'", } func TestUnquote(t *testing.T) { @@ -211,3 +215,15 @@ func TestUnquote(t *testing.T) { } } } + +func BenchmarkUnquoteEasy(b *testing.B) { + for i := 0; i < b.N; i++ { + Unquote(`"Give me a rock, paper and scissors and I will move the world."`) + } +} + +func BenchmarkUnquoteHard(b *testing.B) { + for i := 0; i < b.N; i++ { + Unquote(`"\x47ive me a \x72ock, \x70aper and \x73cissors and \x49 will move the world."`) + } +} diff --git a/libgo/go/strings/export_test.go b/libgo/go/strings/export_test.go new file mode 100644 index 0000000..dcfec51 --- /dev/null +++ b/libgo/go/strings/export_test.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strings + +func (r *Replacer) Replacer() interface{} { + return r.r +} diff --git a/libgo/go/strings/replace.go b/libgo/go/strings/replace.go new file mode 100644 index 0000000..64a7f20 --- /dev/null +++ b/libgo/go/strings/replace.go @@ -0,0 +1,315 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strings + +import ( + "io" + "os" +) + +// A Replacer replaces a list of strings with replacements. +type Replacer struct { + r replacer +} + +// replacer is the interface that a replacement algorithm needs to implement. +type replacer interface { + Replace(s string) string + WriteString(w io.Writer, s string) (n int, err os.Error) +} + +// byteBitmap represents bytes which are sought for replacement. +// byteBitmap is 256 bits wide, with a bit set for each old byte to be +// replaced. +type byteBitmap [256 / 32]uint32 + +func (m *byteBitmap) set(b byte) { + m[b>>5] |= uint32(1 << (b & 31)) +} + +// NewReplacer returns a new Replacer from a list of old, new string pairs. +// Replacements are performed in order, without overlapping matches. +func NewReplacer(oldnew ...string) *Replacer { + if len(oldnew)%2 == 1 { + panic("strings.NewReplacer: odd argument count") + } + + // Possible implementations. + var ( + bb byteReplacer + bs byteStringReplacer + gen genericReplacer + ) + + allOldBytes, allNewBytes := true, true + for len(oldnew) > 0 { + old, new := oldnew[0], oldnew[1] + oldnew = oldnew[2:] + if len(old) != 1 { + allOldBytes = false + } + if len(new) != 1 { + allNewBytes = false + } + + // generic + gen.p = append(gen.p, pair{old, new}) + + // byte -> string + if allOldBytes { + bs.old.set(old[0]) + bs.new[old[0]] = []byte(new) + } + + // byte -> byte + if allOldBytes && allNewBytes { + bb.old.set(old[0]) + bb.new[old[0]] = new[0] + } + } + + if allOldBytes && allNewBytes { + return &Replacer{r: &bb} + } + if allOldBytes { + return &Replacer{r: &bs} + } + return &Replacer{r: &gen} +} + +// Replace returns a copy of s with all replacements performed. +func (r *Replacer) Replace(s string) string { + return r.r.Replace(s) +} + +// WriteString writes s to w with all replacements performed. +func (r *Replacer) WriteString(w io.Writer, s string) (n int, err os.Error) { + return r.r.WriteString(w, s) +} + +// genericReplacer is the fully generic (and least optimized) algorithm. +// It's used as a fallback when nothing faster can be used. +type genericReplacer struct { + p []pair +} + +type pair struct{ old, new string } + +type appendSliceWriter struct { + b []byte +} + +func (w *appendSliceWriter) Write(p []byte) (int, os.Error) { + w.b = append(w.b, p...) + return len(p), nil +} + +func (r *genericReplacer) Replace(s string) string { + // TODO(bradfitz): optimized version + n, _ := r.WriteString(discard, s) + w := appendSliceWriter{make([]byte, 0, n)} + r.WriteString(&w, s) + return string(w.b) +} + +func (r *genericReplacer) WriteString(w io.Writer, s string) (n int, err os.Error) { + lastEmpty := false // the last replacement was of the empty string +Input: + // TODO(bradfitz): optimized version + for i := 0; i < len(s); { + for _, p := range r.p { + if p.old == "" && lastEmpty { + // Don't let old match twice in a row. + // (it doesn't advance the input and + // would otherwise loop forever) + continue + } + if HasPrefix(s[i:], p.old) { + if p.new != "" { + wn, err := w.Write([]byte(p.new)) + n += wn + if err != nil { + return n, err + } + } + i += len(p.old) + lastEmpty = p.old == "" + continue Input + } + } + wn, err := w.Write([]byte{s[i]}) + n += wn + if err != nil { + return n, err + } + i++ + } + + // Final empty match at end. + for _, p := range r.p { + if p.old == "" { + if p.new != "" { + wn, err := w.Write([]byte(p.new)) + n += wn + if err != nil { + return n, err + } + } + break + } + } + + return n, nil +} + +// byteReplacer is the implementation that's used when all the "old" +// and "new" values are single ASCII bytes. +type byteReplacer struct { + // old has a bit set for each old byte that should be replaced. + old byteBitmap + + // replacement byte, indexed by old byte. only valid if + // corresponding old bit is set. + new [256]byte +} + +func (r *byteReplacer) Replace(s string) string { + var buf []byte // lazily allocated + for i := 0; i < len(s); i++ { + b := s[i] + if r.old[b>>5]&uint32(1<<(b&31)) != 0 { + if buf == nil { + buf = []byte(s) + } + buf[i] = r.new[b] + } + } + if buf == nil { + return s + } + return string(buf) +} + +func (r *byteReplacer) WriteString(w io.Writer, s string) (n int, err os.Error) { + // TODO(bradfitz): use io.WriteString with slices of s, avoiding allocation. + bufsize := 32 << 10 + if len(s) < bufsize { + bufsize = len(s) + } + buf := make([]byte, bufsize) + + for len(s) > 0 { + ncopy := copy(buf, s[:]) + s = s[ncopy:] + for i, b := range buf[:ncopy] { + if r.old[b>>5]&uint32(1<<(b&31)) != 0 { + buf[i] = r.new[b] + } + } + wn, err := w.Write(buf[:ncopy]) + n += wn + if err != nil { + return n, err + } + } + return n, nil +} + +// byteStringReplacer is the implementation that's used when all the +// "old" values are single ASCII bytes but the "new" values vary in +// size. +type byteStringReplacer struct { + // old has a bit set for each old byte that should be replaced. + old byteBitmap + + // replacement string, indexed by old byte. only valid if + // corresponding old bit is set. + new [256][]byte +} + +func (r *byteStringReplacer) Replace(s string) string { + newSize := 0 + anyChanges := false + for i := 0; i < len(s); i++ { + b := s[i] + if r.old[b>>5]&uint32(1<<(b&31)) != 0 { + anyChanges = true + newSize += len(r.new[b]) + } else { + newSize++ + } + } + if !anyChanges { + return s + } + buf := make([]byte, newSize) + bi := buf + for i := 0; i < len(s); i++ { + b := s[i] + if r.old[b>>5]&uint32(1<<(b&31)) != 0 { + n := copy(bi[:], r.new[b]) + bi = bi[n:] + } else { + bi[0] = b + bi = bi[1:] + } + } + return string(buf) +} + +// WriteString maintains one buffer that's at most 32KB. The bytes in +// s are enumerated and the buffer is filled. If it reaches its +// capacity or a byte has a replacement, the buffer is flushed to w. +func (r *byteStringReplacer) WriteString(w io.Writer, s string) (n int, err os.Error) { + // TODO(bradfitz): use io.WriteString with slices of s instead. + bufsize := 32 << 10 + if len(s) < bufsize { + bufsize = len(s) + } + buf := make([]byte, bufsize) + bi := buf[:0] + + for i := 0; i < len(s); i++ { + b := s[i] + var new []byte + if r.old[b>>5]&uint32(1<<(b&31)) != 0 { + new = r.new[b] + } else { + bi = append(bi, b) + } + if len(bi) == cap(bi) || (len(bi) > 0 && len(new) > 0) { + nw, err := w.Write(bi) + n += nw + if err != nil { + return n, err + } + bi = buf[:0] + } + if len(new) > 0 { + nw, err := w.Write(new) + n += nw + if err != nil { + return n, err + } + } + } + if len(bi) > 0 { + nw, err := w.Write(bi) + n += nw + if err != nil { + return n, err + } + } + return n, nil +} + +// strings is too low-level to import io/ioutil +var discard io.Writer = devNull(0) + +type devNull int + +func (devNull) Write(p []byte) (int, os.Error) { + return len(p), nil +} diff --git a/libgo/go/strings/replace_test.go b/libgo/go/strings/replace_test.go new file mode 100644 index 0000000..e337856 --- /dev/null +++ b/libgo/go/strings/replace_test.go @@ -0,0 +1,174 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strings_test + +import ( + "bytes" + "fmt" + "log" + . "strings" + "testing" +) + +var _ = log.Printf + +type ReplacerTest struct { + r *Replacer + in string + out string +} + +var htmlEscaper = NewReplacer("&", "&", "<", "<", ">", ">", "\"", """) + +// The http package's old HTML escaping function. +func oldhtmlEscape(s string) string { + s = Replace(s, "&", "&", -1) + s = Replace(s, "<", "<", -1) + s = Replace(s, ">", ">", -1) + s = Replace(s, "\"", """, -1) + s = Replace(s, "'", "'", -1) + return s +} + +var replacer = NewReplacer("aaa", "3[aaa]", "aa", "2[aa]", "a", "1[a]", "i", "i", + "longerst", "most long", "longer", "medium", "long", "short", + "X", "Y", "Y", "Z") + +var capitalLetters = NewReplacer("a", "A", "b", "B") + +var blankToXReplacer = NewReplacer("", "X", "o", "O") + +var ReplacerTests = []ReplacerTest{ + // byte->string + {htmlEscaper, "No changes", "No changes"}, + {htmlEscaper, "I <3 escaping & stuff", "I <3 escaping & stuff"}, + {htmlEscaper, "&&&", "&&&"}, + + // generic + {replacer, "fooaaabar", "foo3[aaa]b1[a]r"}, + {replacer, "long, longerst, longer", "short, most long, medium"}, + {replacer, "XiX", "YiY"}, + + // byte->byte + {capitalLetters, "brad", "BrAd"}, + {capitalLetters, Repeat("a", (32<<10)+123), Repeat("A", (32<<10)+123)}, + + // hitting "" special case + {blankToXReplacer, "oo", "XOXOX"}, +} + +func TestReplacer(t *testing.T) { + for i, tt := range ReplacerTests { + if s := tt.r.Replace(tt.in); s != tt.out { + t.Errorf("%d. Replace(%q) = %q, want %q", i, tt.in, s, tt.out) + } + var buf bytes.Buffer + n, err := tt.r.WriteString(&buf, tt.in) + if err != nil { + t.Errorf("%d. WriteString: %v", i, err) + continue + } + got := buf.String() + if got != tt.out { + t.Errorf("%d. WriteString(%q) wrote %q, want %q", i, tt.in, got, tt.out) + continue + } + if n != len(tt.out) { + t.Errorf("%d. WriteString(%q) wrote correct string but reported %d bytes; want %d (%q)", + i, tt.in, n, len(tt.out), tt.out) + } + } +} + +// pickAlgorithmTest is a test that verifies that given input for a +// Replacer that we pick the correct algorithm. +type pickAlgorithmTest struct { + r *Replacer + want string // name of algorithm +} + +var pickAlgorithmTests = []pickAlgorithmTest{ + {capitalLetters, "*strings.byteReplacer"}, + {NewReplacer("12", "123"), "*strings.genericReplacer"}, + {NewReplacer("1", "12"), "*strings.byteStringReplacer"}, + {htmlEscaper, "*strings.byteStringReplacer"}, +} + +func TestPickAlgorithm(t *testing.T) { + for i, tt := range pickAlgorithmTests { + got := fmt.Sprintf("%T", tt.r.Replacer()) + if got != tt.want { + t.Errorf("%d. algorithm = %s, want %s", i, got, tt.want) + } + } +} + +func BenchmarkGenericMatch(b *testing.B) { + str := Repeat("A", 100) + Repeat("B", 100) + generic := NewReplacer("a", "A", "b", "B", "12", "123") // varying lengths forces generic + for i := 0; i < b.N; i++ { + generic.Replace(str) + } +} + +func BenchmarkByteByteNoMatch(b *testing.B) { + str := Repeat("A", 100) + Repeat("B", 100) + for i := 0; i < b.N; i++ { + capitalLetters.Replace(str) + } +} + +func BenchmarkByteByteMatch(b *testing.B) { + str := Repeat("a", 100) + Repeat("b", 100) + for i := 0; i < b.N; i++ { + capitalLetters.Replace(str) + } +} + +func BenchmarkByteStringMatch(b *testing.B) { + str := "<" + Repeat("a", 99) + Repeat("b", 99) + ">" + for i := 0; i < b.N; i++ { + htmlEscaper.Replace(str) + } +} + +func BenchmarkHTMLEscapeNew(b *testing.B) { + str := "I <3 to escape HTML & other text too." + for i := 0; i < b.N; i++ { + htmlEscaper.Replace(str) + } +} + +func BenchmarkHTMLEscapeOld(b *testing.B) { + str := "I <3 to escape HTML & other text too." + for i := 0; i < b.N; i++ { + oldhtmlEscape(str) + } +} + +// BenchmarkByteByteReplaces compares byteByteImpl against multiple Replaces. +func BenchmarkByteByteReplaces(b *testing.B) { + str := Repeat("a", 100) + Repeat("b", 100) + for i := 0; i < b.N; i++ { + Replace(Replace(str, "a", "A", -1), "b", "B", -1) + } +} + +// BenchmarkByteByteMap compares byteByteImpl against Map. +func BenchmarkByteByteMap(b *testing.B) { + str := Repeat("a", 100) + Repeat("b", 100) + fn := func(r int) int { + switch r { + case 'a': + return int('A') + case 'b': + return int('B') + } + return r + } + for i := 0; i < b.N; i++ { + Map(fn, str) + } +} diff --git a/libgo/go/strings/strings.go b/libgo/go/strings/strings.go index c547297..58301fe 100644 --- a/libgo/go/strings/strings.go +++ b/libgo/go/strings/strings.go @@ -583,3 +583,58 @@ func Replace(s, old, new string, n int) string { w += copy(t[w:], s[start:]) return string(t[0:w]) } + +// EqualFold reports whether s and t, interpreted as UTF-8 strings, +// are equal under Unicode case-folding. +func EqualFold(s, t string) bool { + for s != "" && t != "" { + // Extract first rune from each string. + var sr, tr int + if s[0] < utf8.RuneSelf { + sr, s = int(s[0]), s[1:] + } else { + r, size := utf8.DecodeRuneInString(s) + sr, s = r, s[size:] + } + if t[0] < utf8.RuneSelf { + tr, t = int(t[0]), t[1:] + } else { + r, size := utf8.DecodeRuneInString(t) + tr, t = r, t[size:] + } + + // If they match, keep going; if not, return false. + + // Easy case. + if tr == sr { + continue + } + + // Make sr < tr to simplify what follows. + if tr < sr { + tr, sr = sr, tr + } + // Fast check for ASCII. + if tr < utf8.RuneSelf && 'A' <= sr && sr <= 'Z' { + // ASCII, and sr is upper case. tr must be lower case. + if tr == sr+'a'-'A' { + continue + } + return false + } + + // General case. SimpleFold(x) returns the next equivalent rune > x + // or wraps around to smaller values. + r := unicode.SimpleFold(sr) + for r != sr && r < tr { + r = unicode.SimpleFold(r) + } + if r == tr { + continue + } + return false + } + + // One string is empty. Are both? + return s == t +} diff --git a/libgo/go/strings/strings_test.go b/libgo/go/strings/strings_test.go index 409d4da..0859ddd 100644 --- a/libgo/go/strings/strings_test.go +++ b/libgo/go/strings/strings_test.go @@ -120,13 +120,11 @@ func TestLastIndex(t *testing.T) { runIndexTests(t, LastIndex, "LastIndex", l func TestIndexAny(t *testing.T) { runIndexTests(t, IndexAny, "IndexAny", indexAnyTests) } func TestLastIndexAny(t *testing.T) { runIndexTests(t, LastIndexAny, "LastIndexAny", lastIndexAnyTests) } -type IndexRuneTest struct { +var indexRuneTests = []struct { s string rune int out int -} - -var indexRuneTests = []IndexRuneTest{ +}{ {"a A x", 'A', 2}, {"some_text=some_value", '=', 9}, {"☺a", 'a', 3}, @@ -170,13 +168,11 @@ func BenchmarkIndex(b *testing.B) { } } -type ExplodeTest struct { +var explodetests = []struct { s string n int a []string -} - -var explodetests = []ExplodeTest{ +}{ {"", -1, []string{}}, {abcd, 4, []string{"a", "b", "c", "d"}}, {faces, 3, []string{"☺", "☻", "☹"}}, @@ -308,15 +304,16 @@ func TestFields(t *testing.T) { } } +var FieldsFuncTests = []FieldsTest{ + {"", []string{}}, + {"XX", []string{}}, + {"XXhiXXX", []string{"hi"}}, + {"aXXbXXXcX", []string{"a", "b", "c"}}, +} + func TestFieldsFunc(t *testing.T) { pred := func(c int) bool { return c == 'X' } - var fieldsFuncTests = []FieldsTest{ - {"", []string{}}, - {"XX", []string{}}, - {"XXhiXXX", []string{"hi"}}, - {"aXXbXXXcX", []string{"a", "b", "c"}}, - } - for _, tt := range fieldsFuncTests { + for _, tt := range FieldsFuncTests { a := FieldsFunc(tt.s, pred) if !eq(a, tt.a) { t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) @@ -491,12 +488,10 @@ func TestSpecialCase(t *testing.T) { func TestTrimSpace(t *testing.T) { runStringTests(t, TrimSpace, "TrimSpace", trimSpaceTests) } -type TrimTest struct { +var trimTests = []struct { f func(string, string) string in, cutset, out string -} - -var trimTests = []TrimTest{ +}{ {Trim, "abba", "a", "bb"}, {Trim, "abba", "ab", ""}, {TrimLeft, "abba", "ab", ""}, @@ -555,11 +550,6 @@ var isValidRune = predicate{ "IsValidRune", } -type TrimFuncTest struct { - f predicate - in, out string -} - func not(p predicate) predicate { return predicate{ func(r int) bool { @@ -569,7 +559,10 @@ func not(p predicate) predicate { } } -var trimFuncTests = []TrimFuncTest{ +var trimFuncTests = []struct { + f predicate + in, out string +}{ {isSpace, space + " hello " + space, "hello"}, {isDigit, "\u0e50\u0e5212hello34\u0e50\u0e51", "hello"}, {isUpper, "\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", "hello"}, @@ -588,13 +581,11 @@ func TestTrimFunc(t *testing.T) { } } -type IndexFuncTest struct { +var indexFuncTests = []struct { in string f predicate first, last int -} - -var indexFuncTests = []IndexFuncTest{ +}{ {"", isValidRune, -1, -1}, {"abc", isDigit, -1, -1}, {"0123", isDigit, 0, 3}, @@ -692,12 +683,10 @@ func TestCaseConsistency(t *testing.T) { */ } -type RepeatTest struct { +var RepeatTests = []struct { in, out string count int -} - -var RepeatTests = []RepeatTest{ +}{ {"", "", 0}, {"", "", 1}, {"", "", 2}, @@ -729,13 +718,11 @@ func runesEqual(a, b []int) bool { return true } -type RunesTest struct { +var RunesTests = []struct { in string out []int lossy bool -} - -var RunesTests = []RunesTest{ +}{ {"", []int{}, false}, {" ", []int{32}, false}, {"ABC", []int{65, 66, 67}, false}, @@ -846,14 +833,12 @@ func TestReadRune(t *testing.T) { } } -type ReplaceTest struct { +var ReplaceTests = []struct { in string old, new string n int out string -} - -var ReplaceTests = []ReplaceTest{ +}{ {"hello", "l", "L", 0, "hello"}, {"hello", "l", "L", -1, "heLLo"}, {"hello", "x", "X", -1, "hello"}, @@ -883,11 +868,9 @@ func TestReplace(t *testing.T) { } } -type TitleTest struct { +var TitleTests = []struct { in, out string -} - -var TitleTests = []TitleTest{ +}{ {"", ""}, {"a", "A"}, {" aaa aaa aaa ", " Aaa Aaa Aaa "}, @@ -905,12 +888,10 @@ func TestTitle(t *testing.T) { } } -type ContainsTest struct { +var ContainsTests = []struct { str, substr string expected bool -} - -var ContainsTests = []ContainsTest{ +}{ {"abc", "bc", true}, {"abc", "bcd", false}, {"abc", "", true}, @@ -925,3 +906,31 @@ func TestContains(t *testing.T) { } } } + +var EqualFoldTests = []struct { + s, t string + out bool +}{ + {"abc", "abc", true}, + {"ABcd", "ABcd", true}, + {"123abc", "123ABC", true}, + {"αβδ", "ΑΒΔ", true}, + {"abc", "xyz", false}, + {"abc", "XYZ", false}, + {"abcdefghijk", "abcdefghijX", false}, + {"abcdefghijk", "abcdefghij\u212A", true}, + {"abcdefghijK", "abcdefghij\u212A", true}, + {"abcdefghijkz", "abcdefghij\u212Ay", false}, + {"abcdefghijKz", "abcdefghij\u212Ay", false}, +} + +func TestEqualFold(t *testing.T) { + for _, tt := range EqualFoldTests { + if out := EqualFold(tt.s, tt.t); out != tt.out { + t.Errorf("EqualFold(%#q, %#q) = %v, want %v", tt.s, tt.t, out, tt.out) + } + if out := EqualFold(tt.t, tt.s); out != tt.out { + t.Errorf("EqualFold(%#q, %#q) = %v, want %v", tt.t, tt.s, out, tt.out) + } + } +} diff --git a/libgo/go/sync/atomic/atomic.c b/libgo/go/sync/atomic/atomic.c index 6660a7d..e5de5ee 100644 --- a/libgo/go/sync/atomic/atomic.c +++ b/libgo/go/sync/atomic/atomic.c @@ -51,6 +51,15 @@ CompareAndSwapUintptr (uintptr_t *val, uintptr_t old, uintptr_t new) return __sync_bool_compare_and_swap (val, old, new); } +_Bool CompareAndSwapPointer (void **, void *, void *) + asm ("libgo_sync.atomic.CompareAndSwapPointer"); + +_Bool +CompareAndSwapPointer (void **val, void *old, void *new) +{ + return __sync_bool_compare_and_swap (val, old, new); +} + int32_t AddInt32 (int32_t *, int32_t) asm ("libgo_sync.atomic.AddInt32"); @@ -110,6 +119,20 @@ LoadInt32 (int32_t *addr) return v; } +int64_t LoadInt64 (int64_t *addr) + asm ("libgo_sync.atomic.LoadInt64"); + +int64_t +LoadInt64 (int64_t *addr) +{ + int64_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, v)) + v = *addr; + return v; +} + uint32_t LoadUint32 (uint32_t *addr) asm ("libgo_sync.atomic.LoadUint32"); @@ -123,3 +146,123 @@ LoadUint32 (uint32_t *addr) v = *addr; return v; } + +uint64_t LoadUint64 (uint64_t *addr) + asm ("libgo_sync.atomic.LoadUint64"); + +uint64_t +LoadUint64 (uint64_t *addr) +{ + uint64_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, v)) + v = *addr; + return v; +} + +uintptr_t LoadUintptr (uintptr_t *addr) + asm ("libgo_sync.atomic.LoadUintptr"); + +uintptr_t +LoadUintptr (uintptr_t *addr) +{ + uintptr_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, v)) + v = *addr; + return v; +} + +void *LoadPointer (void **addr) + asm ("libgo_sync.atomic.LoadPointer"); + +void * +LoadPointer (void **addr) +{ + void *v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, v)) + v = *addr; + return v; +} + +void StoreInt32 (int32_t *addr, int32_t val) + asm ("libgo_sync.atomic.StoreInt32"); + +void +StoreInt32 (int32_t *addr, int32_t val) +{ + int32_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, val)) + v = *addr; +} + +void StoreInt64 (int64_t *addr, int64_t val) + asm ("libgo_sync.atomic.StoreInt64"); + +void +StoreInt64 (int64_t *addr, int64_t val) +{ + int64_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, val)) + v = *addr; +} + +void StoreUint32 (uint32_t *addr, uint32_t val) + asm ("libgo_sync.atomic.StoreUint32"); + +void +StoreUint32 (uint32_t *addr, uint32_t val) +{ + uint32_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, val)) + v = *addr; +} + +void StoreUint64 (uint64_t *addr, uint64_t val) + asm ("libgo_sync.atomic.StoreUint64"); + +void +StoreUint64 (uint64_t *addr, uint64_t val) +{ + uint64_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, val)) + v = *addr; +} + +void StoreUintptr (uintptr_t *addr, uintptr_t val) + asm ("libgo_sync.atomic.StoreUintptr"); + +void +StoreUintptr (uintptr_t *addr, uintptr_t val) +{ + uintptr_t v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, val)) + v = *addr; +} + +void StorePointer (void **addr, void *val) + asm ("libgo_sync.atomic.StorePointer"); + +void +StorePointer (void **addr, void *val) +{ + void *v; + + v = *addr; + while (! __sync_bool_compare_and_swap (addr, v, val)) + v = *addr; +} diff --git a/libgo/go/sync/atomic/atomic_test.go b/libgo/go/sync/atomic/atomic_test.go index 2229e58..02ee24b 100644 --- a/libgo/go/sync/atomic/atomic_test.go +++ b/libgo/go/sync/atomic/atomic_test.go @@ -164,17 +164,17 @@ func TestCompareAndSwapInt32(t *testing.T) { for val := int32(1); val+val > val; val += val { x.i = val if !CompareAndSwapInt32(&x.i, val, val+1) { - t.Errorf("should have swapped %#x %#x", val, val+1) + t.Fatalf("should have swapped %#x %#x", val, val+1) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } x.i = val + 1 if CompareAndSwapInt32(&x.i, val, val+2) { - t.Errorf("should not have swapped %#x %#x", val, val+2) + t.Fatalf("should not have swapped %#x %#x", val, val+2) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } } if x.before != magic32 || x.after != magic32 { @@ -193,17 +193,17 @@ func TestCompareAndSwapUint32(t *testing.T) { for val := uint32(1); val+val > val; val += val { x.i = val if !CompareAndSwapUint32(&x.i, val, val+1) { - t.Errorf("should have swapped %#x %#x", val, val+1) + t.Fatalf("should have swapped %#x %#x", val, val+1) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } x.i = val + 1 if CompareAndSwapUint32(&x.i, val, val+2) { - t.Errorf("should not have swapped %#x %#x", val, val+2) + t.Fatalf("should not have swapped %#x %#x", val, val+2) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } } if x.before != magic32 || x.after != magic32 { @@ -226,17 +226,17 @@ func TestCompareAndSwapInt64(t *testing.T) { for val := int64(1); val+val > val; val += val { x.i = val if !CompareAndSwapInt64(&x.i, val, val+1) { - t.Errorf("should have swapped %#x %#x", val, val+1) + t.Fatalf("should have swapped %#x %#x", val, val+1) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } x.i = val + 1 if CompareAndSwapInt64(&x.i, val, val+2) { - t.Errorf("should not have swapped %#x %#x", val, val+2) + t.Fatalf("should not have swapped %#x %#x", val, val+2) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } } if x.before != magic64 || x.after != magic64 { @@ -259,17 +259,17 @@ func TestCompareAndSwapUint64(t *testing.T) { for val := uint64(1); val+val > val; val += val { x.i = val if !CompareAndSwapUint64(&x.i, val, val+1) { - t.Errorf("should have swapped %#x %#x", val, val+1) + t.Fatalf("should have swapped %#x %#x", val, val+1) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } x.i = val + 1 if CompareAndSwapUint64(&x.i, val, val+2) { - t.Errorf("should not have swapped %#x %#x", val, val+2) + t.Fatalf("should not have swapped %#x %#x", val, val+2) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } } if x.before != magic64 || x.after != magic64 { @@ -290,17 +290,48 @@ func TestCompareAndSwapUintptr(t *testing.T) { for val := uintptr(1); val+val > val; val += val { x.i = val if !CompareAndSwapUintptr(&x.i, val, val+1) { - t.Errorf("should have swapped %#x %#x", val, val+1) + t.Fatalf("should have swapped %#x %#x", val, val+1) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } x.i = val + 1 if CompareAndSwapUintptr(&x.i, val, val+2) { - t.Errorf("should not have swapped %#x %#x", val, val+2) + t.Fatalf("should not have swapped %#x %#x", val, val+2) } if x.i != val+1 { - t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +func TestCompareAndSwapPointer(t *testing.T) { + var x struct { + before uintptr + i unsafe.Pointer + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for val := uintptr(1); val+val > val; val += val { + x.i = unsafe.Pointer(val) + if !CompareAndSwapPointer(&x.i, unsafe.Pointer(val), unsafe.Pointer(val+1)) { + t.Fatalf("should have swapped %#x %#x", val, val+1) + } + if x.i != unsafe.Pointer(val+1) { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = unsafe.Pointer(val + 1) + if CompareAndSwapPointer(&x.i, unsafe.Pointer(val), unsafe.Pointer(val+2)) { + t.Fatalf("should not have swapped %#x %#x", val, val+2) + } + if x.i != unsafe.Pointer(val+1) { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) } } if x.before != magicptr || x.after != magicptr { @@ -348,6 +379,236 @@ func TestLoadUint32(t *testing.T) { } } +func TestLoadInt64(t *testing.T) { + if test64err != nil { + t.Logf("Skipping 64-bit tests: %v", test64err) + return + } + var x struct { + before int64 + i int64 + after int64 + } + x.before = magic64 + x.after = magic64 + for delta := int64(1); delta+delta > delta; delta += delta { + k := LoadInt64(&x.i) + if k != x.i { + t.Fatalf("delta=%d i=%d k=%d", delta, x.i, k) + } + x.i += delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestLoadUint64(t *testing.T) { + if test64err != nil { + t.Logf("Skipping 64-bit tests: %v", test64err) + return + } + var x struct { + before uint64 + i uint64 + after uint64 + } + x.before = magic64 + x.after = magic64 + for delta := uint64(1); delta+delta > delta; delta += delta { + k := LoadUint64(&x.i) + if k != x.i { + t.Fatalf("delta=%d i=%d k=%d", delta, x.i, k) + } + x.i += delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestLoadUintptr(t *testing.T) { + var x struct { + before uintptr + i uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := LoadUintptr(&x.i) + if k != x.i { + t.Fatalf("delta=%d i=%d k=%d", delta, x.i, k) + } + x.i += delta + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +func TestLoadPointer(t *testing.T) { + var x struct { + before uintptr + i unsafe.Pointer + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := LoadPointer(&x.i) + if k != x.i { + t.Fatalf("delta=%d i=%d k=%d", delta, x.i, k) + } + x.i = unsafe.Pointer(uintptr(x.i) + delta) + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +func TestStoreInt32(t *testing.T) { + var x struct { + before int32 + i int32 + after int32 + } + x.before = magic32 + x.after = magic32 + v := int32(0) + for delta := int32(1); delta+delta > delta; delta += delta { + StoreInt32(&x.i, v) + if x.i != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v) + } + v += delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestStoreUint32(t *testing.T) { + var x struct { + before uint32 + i uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + v := uint32(0) + for delta := uint32(1); delta+delta > delta; delta += delta { + StoreUint32(&x.i, v) + if x.i != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v) + } + v += delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestStoreInt64(t *testing.T) { + if test64err != nil { + t.Logf("Skipping 64-bit tests: %v", test64err) + return + } + var x struct { + before int64 + i int64 + after int64 + } + x.before = magic64 + x.after = magic64 + v := int64(0) + for delta := int64(1); delta+delta > delta; delta += delta { + StoreInt64(&x.i, v) + if x.i != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v) + } + v += delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestStoreUint64(t *testing.T) { + if test64err != nil { + t.Logf("Skipping 64-bit tests: %v", test64err) + return + } + var x struct { + before uint64 + i uint64 + after uint64 + } + x.before = magic64 + x.after = magic64 + v := uint64(0) + for delta := uint64(1); delta+delta > delta; delta += delta { + StoreUint64(&x.i, v) + if x.i != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v) + } + v += delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestStoreUintptr(t *testing.T) { + var x struct { + before uintptr + i uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + v := uintptr(0) + for delta := uintptr(1); delta+delta > delta; delta += delta { + StoreUintptr(&x.i, v) + if x.i != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v) + } + v += delta + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +func TestStorePointer(t *testing.T) { + var x struct { + before uintptr + i unsafe.Pointer + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + v := unsafe.Pointer(uintptr(0)) + for delta := uintptr(1); delta+delta > delta; delta += delta { + StorePointer(&x.i, unsafe.Pointer(v)) + if x.i != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v) + } + v = unsafe.Pointer(uintptr(v) + delta) + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + // Tests of correct behavior, with contention. // (Is the function atomic?) // @@ -366,6 +627,7 @@ var hammer32 = []struct { {"CompareAndSwapInt32", hammerCompareAndSwapInt32}, {"CompareAndSwapUint32", hammerCompareAndSwapUint32}, {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr32}, + {"CompareAndSwapPointer", hammerCompareAndSwapPointer32}, } func init() { @@ -436,6 +698,20 @@ func hammerCompareAndSwapUintptr32(uval *uint32, count int) { } } +func hammerCompareAndSwapPointer32(uval *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + val := (*unsafe.Pointer)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapPointer(val, v, unsafe.Pointer(uintptr(v)+1)) { + break + } + } + } +} + func TestHammer32(t *testing.T) { const p = 4 n := 100000 @@ -460,7 +736,7 @@ func TestHammer32(t *testing.T) { <-c } if val != uint32(n)*p { - t.Errorf("%s: val=%d want %d", tt.name, val, n*p) + t.Fatalf("%s: val=%d want %d", tt.name, val, n*p) } } } @@ -475,6 +751,7 @@ var hammer64 = []struct { {"CompareAndSwapInt64", hammerCompareAndSwapInt64}, {"CompareAndSwapUint64", hammerCompareAndSwapUint64}, {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr64}, + {"CompareAndSwapPointer", hammerCompareAndSwapPointer64}, } func init() { @@ -545,6 +822,20 @@ func hammerCompareAndSwapUintptr64(uval *uint64, count int) { } } +func hammerCompareAndSwapPointer64(uval *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + val := (*unsafe.Pointer)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapPointer(val, v, unsafe.Pointer(uintptr(v)+1)) { + break + } + } + } +} + func TestHammer64(t *testing.T) { if test64err != nil { t.Logf("Skipping 64-bit tests: %v", test64err) @@ -573,63 +864,141 @@ func TestHammer64(t *testing.T) { <-c } if val != uint64(n)*p { - t.Errorf("%s: val=%d want %d", tt.name, val, n*p) + t.Fatalf("%s: val=%d want %d", tt.name, val, n*p) } } } -func hammerLoadInt32(t *testing.T, uval *uint32) { - val := (*int32)(unsafe.Pointer(uval)) - for { - v := LoadInt32(val) +func hammerStoreLoadInt32(t *testing.T, valp unsafe.Pointer) { + val := (*int32)(valp) + v := LoadInt32(val) + vlo := v & ((1 << 16) - 1) + vhi := v >> 16 + if vlo != vhi { + t.Fatalf("Int32: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<16 + if vlo == 1e4 { + new = 0 + } + StoreInt32(val, new) +} + +func hammerStoreLoadUint32(t *testing.T, valp unsafe.Pointer) { + val := (*uint32)(valp) + v := LoadUint32(val) + vlo := v & ((1 << 16) - 1) + vhi := v >> 16 + if vlo != vhi { + t.Fatalf("Uint32: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<16 + if vlo == 1e4 { + new = 0 + } + StoreUint32(val, new) +} + +func hammerStoreLoadInt64(t *testing.T, valp unsafe.Pointer) { + val := (*int64)(valp) + v := LoadInt64(val) + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Int64: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<32 + StoreInt64(val, new) +} + +func hammerStoreLoadUint64(t *testing.T, valp unsafe.Pointer) { + val := (*uint64)(valp) + v := LoadUint64(val) + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Uint64: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<32 + StoreUint64(val, new) +} + +func hammerStoreLoadUintptr(t *testing.T, valp unsafe.Pointer) { + val := (*uintptr)(valp) + var test64 uint64 = 1 << 50 + arch32 := uintptr(test64) == 0 + v := LoadUintptr(val) + new := v + if arch32 { vlo := v & ((1 << 16) - 1) vhi := v >> 16 if vlo != vhi { - t.Fatalf("LoadInt32: %#x != %#x", vlo, vhi) + t.Fatalf("Uintptr: %#x != %#x", vlo, vhi) } - new := v + 1 + 1<<16 + new = v + 1 + 1<<16 if vlo == 1e4 { new = 0 } - if CompareAndSwapInt32(val, v, new) { - break + } else { + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Uintptr: %#x != %#x", vlo, vhi) } + inc := uint64(1 + 1<<32) + new = v + uintptr(inc) } + StoreUintptr(val, new) } -func hammerLoadUint32(t *testing.T, val *uint32) { - for { - v := LoadUint32(val) +func hammerStoreLoadPointer(t *testing.T, valp unsafe.Pointer) { + val := (*unsafe.Pointer)(valp) + var test64 uint64 = 1 << 50 + arch32 := uintptr(test64) == 0 + v := uintptr(LoadPointer(val)) + new := v + if arch32 { vlo := v & ((1 << 16) - 1) vhi := v >> 16 if vlo != vhi { - t.Fatalf("LoadUint32: %#x != %#x", vlo, vhi) + t.Fatalf("Pointer: %#x != %#x", vlo, vhi) } - new := v + 1 + 1<<16 + new = v + 1 + 1<<16 if vlo == 1e4 { new = 0 } - if CompareAndSwapUint32(val, v, new) { - break + } else { + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Pointer: %#x != %#x", vlo, vhi) } + inc := uint64(1 + 1<<32) + new = v + uintptr(inc) } + StorePointer(val, unsafe.Pointer(new)) } -func TestHammerLoad(t *testing.T) { - tests := [...]func(*testing.T, *uint32){hammerLoadInt32, hammerLoadUint32} - n := 100000 +func TestHammerStoreLoad(t *testing.T) { + var tests []func(*testing.T, unsafe.Pointer) + tests = append(tests, hammerStoreLoadInt32, hammerStoreLoadUint32, + hammerStoreLoadUintptr, hammerStoreLoadPointer) + if test64err == nil { + tests = append(tests, hammerStoreLoadInt64, hammerStoreLoadUint64) + } + n := int(1e6) if testing.Short() { - n = 10000 + n = int(1e4) } const procs = 8 defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(procs)) for _, tt := range tests { c := make(chan int) - var val uint32 + var val uint64 for p := 0; p < procs; p++ { go func() { for i := 0; i < n; i++ { - tt(t, &val) + tt(t, unsafe.Pointer(&val)) } c <- 1 }() @@ -639,3 +1008,169 @@ func TestHammerLoad(t *testing.T) { } } } + +func TestStoreLoadSeqCst32(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) + N := int32(1e3) + if testing.Short() { + N = int32(1e2) + } + c := make(chan bool, 2) + X := [2]int32{} + ack := [2][3]int32{{-1, -1, -1}, {-1, -1, -1}} + for p := 0; p < 2; p++ { + go func(me int) { + he := 1 - me + for i := int32(1); i < N; i++ { + StoreInt32(&X[me], i) + my := LoadInt32(&X[he]) + StoreInt32(&ack[me][i%3], my) + for w := 1; LoadInt32(&ack[he][i%3]) == -1; w++ { + if w%1000 == 0 { + runtime.Gosched() + } + } + his := LoadInt32(&ack[he][i%3]) + if (my != i && my != i-1) || (his != i && his != i-1) { + t.Fatalf("invalid values: %d/%d (%d)", my, his, i) + } + if my != i && his != i { + t.Fatalf("store/load are not sequentially consistent: %d/%d (%d)", my, his, i) + } + ack[me][(i-1)%3] = -1 + } + c <- true + }(p) + } + <-c + <-c +} + +func TestStoreLoadSeqCst64(t *testing.T) { + if test64err != nil { + t.Logf("Skipping 64-bit tests: %v", test64err) + return + } + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) + N := int64(1e3) + if testing.Short() { + N = int64(1e2) + } + c := make(chan bool, 2) + X := [2]int64{} + ack := [2][3]int64{{-1, -1, -1}, {-1, -1, -1}} + for p := 0; p < 2; p++ { + go func(me int) { + he := 1 - me + for i := int64(1); i < N; i++ { + StoreInt64(&X[me], i) + my := LoadInt64(&X[he]) + StoreInt64(&ack[me][i%3], my) + for w := 1; LoadInt64(&ack[he][i%3]) == -1; w++ { + if w%1000 == 0 { + runtime.Gosched() + } + } + his := LoadInt64(&ack[he][i%3]) + if (my != i && my != i-1) || (his != i && his != i-1) { + t.Fatalf("invalid values: %d/%d (%d)", my, his, i) + } + if my != i && his != i { + t.Fatalf("store/load are not sequentially consistent: %d/%d (%d)", my, his, i) + } + ack[me][(i-1)%3] = -1 + } + c <- true + }(p) + } + <-c + <-c +} + +func TestStoreLoadRelAcq32(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) + N := int32(1e3) + if testing.Short() { + N = int32(1e2) + } + c := make(chan bool, 2) + type Data struct { + signal int32 + pad1 [128]int8 + data1 int32 + pad2 [128]int8 + data2 float32 + } + var X Data + for p := int32(0); p < 2; p++ { + go func(p int32) { + for i := int32(1); i < N; i++ { + if (i+p)%2 == 0 { + X.data1 = i + X.data2 = float32(i) + StoreInt32(&X.signal, i) + } else { + for w := 1; LoadInt32(&X.signal) != i; w++ { + if w%1000 == 0 { + runtime.Gosched() + } + } + d1 := X.data1 + d2 := X.data2 + if d1 != i || d2 != float32(i) { + t.Fatalf("incorrect data: %d/%d (%d)", d1, d2, i) + } + } + } + c <- true + }(p) + } + <-c + <-c +} + +func TestStoreLoadRelAcq64(t *testing.T) { + if test64err != nil { + t.Logf("Skipping 64-bit tests: %v", test64err) + return + } + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) + N := int64(1e3) + if testing.Short() { + N = int64(1e2) + } + c := make(chan bool, 2) + type Data struct { + signal int64 + pad1 [128]int8 + data1 int64 + pad2 [128]int8 + data2 float64 + } + var X Data + for p := int64(0); p < 2; p++ { + go func(p int64) { + for i := int64(1); i < N; i++ { + if (i+p)%2 == 0 { + X.data1 = i + X.data2 = float64(i) + StoreInt64(&X.signal, i) + } else { + for w := 1; LoadInt64(&X.signal) != i; w++ { + if w%1000 == 0 { + runtime.Gosched() + } + } + d1 := X.data1 + d2 := X.data2 + if d1 != i || d2 != float64(i) { + t.Fatalf("incorrect data: %d/%d (%d)", d1, d2, i) + } + } + } + c <- true + }(p) + } + <-c + <-c +} diff --git a/libgo/go/sync/atomic/doc.go b/libgo/go/sync/atomic/doc.go index b35eb53..ecb4808 100644 --- a/libgo/go/sync/atomic/doc.go +++ b/libgo/go/sync/atomic/doc.go @@ -22,9 +22,13 @@ // package atomic +import ( + "unsafe" +) + // BUG(rsc): On ARM, the 64-bit functions use instructions unavailable before ARM 11. // -// On x86-32, the 64-bit functions use instructions unavailable before the Pentium. +// On x86-32, the 64-bit functions use instructions unavailable before the Pentium MMX. // CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value. func CompareAndSwapInt32(val *int32, old, new int32) (swapped bool) @@ -41,6 +45,9 @@ func CompareAndSwapUint64(val *uint64, old, new uint64) (swapped bool) // CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value. func CompareAndSwapUintptr(val *uintptr, old, new uintptr) (swapped bool) +// CompareAndSwapPointer executes the compare-and-swap operation for a unsafe.Pointer value. +func CompareAndSwapPointer(val *unsafe.Pointer, old, new unsafe.Pointer) (swapped bool) + // AddInt32 atomically adds delta to *val and returns the new value. func AddInt32(val *int32, delta int32) (new int32) @@ -59,9 +66,39 @@ func AddUintptr(val *uintptr, delta uintptr) (new uintptr) // LoadInt32 atomically loads *addr. func LoadInt32(addr *int32) (val int32) +// LoadInt64 atomically loads *addr. +func LoadInt64(addr *int64) (val int64) + // LoadUint32 atomically loads *addr. func LoadUint32(addr *uint32) (val uint32) +// LoadUint64 atomically loads *addr. +func LoadUint64(addr *uint64) (val uint64) + +// LoadUintptr atomically loads *addr. +func LoadUintptr(addr *uintptr) (val uintptr) + +// LoadPointer atomically loads *addr. +func LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer) + +// StoreInt32 atomically stores val into *addr. +func StoreInt32(addr *int32, val int32) + +// StoreInt64 atomically stores val into *addr. +func StoreInt64(addr *int64, val int64) + +// StoreUint32 atomically stores val into *addr. +func StoreUint32(addr *uint32, val uint32) + +// StoreUint64 atomically stores val into *addr. +func StoreUint64(addr *uint64, val uint64) + +// StoreUintptr atomically stores val into *addr. +func StoreUintptr(addr *uintptr, val uintptr) + +// StorePointer atomically stores val into *addr. +func StorePointer(addr *unsafe.Pointer, val unsafe.Pointer) + // Helper for ARM. Linker will discard on other systems func panic64() { panic("sync/atomic: broken 64-bit atomic operations (buggy QEMU)") diff --git a/libgo/go/syscall/exec_unix.go b/libgo/go/syscall/exec_unix.go index 96dfa4d..60e9770 100644 --- a/libgo/go/syscall/exec_unix.go +++ b/libgo/go/syscall/exec_unix.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin freebsd linux openbsd - // Fork, exec, wait, etc. package syscall @@ -87,10 +85,10 @@ import ( // 2) Socket. Does not block. Use the ForkLock. // 3) Accept. If using non-blocking mode, use the ForkLock. // Otherwise, live with the race. -// 4) Open. Can block. Use O_CLOEXEC if available (Linux). +// 4) Open. Can block. Use O_CLOEXEC if available (GNU/Linux). // Otherwise, live with the race. // 5) Dup. Does not block. Use the ForkLock. -// On Linux, could use fcntl F_DUPFD_CLOEXEC +// On GNU/Linux, could use fcntl F_DUPFD_CLOEXEC // instead of the ForkLock, but only for dup(fd, -1). var ForkLock sync.RWMutex @@ -254,8 +252,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr continue } if fd[i] == int(i) { - // Dup2(i, i) won't clear close-on-exec flag on Linux, - // probably not elsewhere either. + // Dup2(i, i) won't clear close-on-exec flag on + // GNU/Linux, probably not elsewhere either. _, err1 = raw_fcntl(fd[i], F_SETFD, 0) if err1 != 0 { goto childerror diff --git a/libgo/go/syscall/libcall_linux.go b/libgo/go/syscall/libcall_linux.go index 58422e1..a78af36 100644 --- a/libgo/go/syscall/libcall_linux.go +++ b/libgo/go/syscall/libcall_linux.go @@ -21,7 +21,7 @@ func Futimesat(dirfd int, path string, tv []Timeval) (errno int) { } func Futimes(fd int, tv []Timeval) (errno int) { - // Believe it or not, this is the best we can do on Linux + // Believe it or not, this is the best we can do on GNU/Linux // (and is what glibc does). return Utimes("/proc/self/fd/"+itoa(fd), tv) } diff --git a/libgo/go/syscall/libcall_posix.go b/libgo/go/syscall/libcall_posix.go index 0baec62..5d7c98f 100644 --- a/libgo/go/syscall/libcall_posix.go +++ b/libgo/go/syscall/libcall_posix.go @@ -73,7 +73,7 @@ func Getgroups() (gids []int, errno int) { return nil, 0 } - // Sanity check group count. Max is 1<<16 on Linux. + // Sanity check group count. Max is 1<<16 on GNU/Linux. if n < 0 || n > 1<<20 { return nil, EINVAL } diff --git a/libgo/go/syscall/lsf_linux.go b/libgo/go/syscall/lsf_linux.go index f2bd2b7..0976688 100644 --- a/libgo/go/syscall/lsf_linux.go +++ b/libgo/go/syscall/lsf_linux.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Linux socket filter +// GNU/Linux socket filter package syscall diff --git a/libgo/go/syscall/socket.go b/libgo/go/syscall/socket.go index 9da8a68..e0218ba 100644 --- a/libgo/go/syscall/socket.go +++ b/libgo/go/syscall/socket.go @@ -296,14 +296,10 @@ func SetsockoptIPMreqn(fd, level, opt int, mreq *IPMreqn) (errno int) { */ -/* FIXME: mksysinfo needs to support IPv6Mreq. - func SetsockoptIPv6Mreq(fd, level, opt int, mreq *IPv6Mreq) (errno int) { return setsockopt(fd, level, opt, (*byte)(unsafe.Pointer(mreq)), Socklen_t(unsafe.Sizeof(*mreq))) } -*/ - func SetsockoptString(fd, level, opt int, s string) (errno int) { return setsockopt(fd, level, opt, (*byte)(unsafe.Pointer(&[]byte(s)[0])), Socklen_t(len(s))) } diff --git a/libgo/go/syscall/socket_irix.go b/libgo/go/syscall/socket_irix.go index 6f2aaaf..8bd55b8 100644 --- a/libgo/go/syscall/socket_irix.go +++ b/libgo/go/syscall/socket_irix.go @@ -52,7 +52,7 @@ func (sa *RawSockaddrUnix) getLen() (int, int) { } // Assume path ends at NUL. - // This is not technically the Linux semantics for + // This is not technically the GNU/Linux semantics for // abstract Unix domain sockets--they are supposed // to be uninterpreted fixed-size binary blobs--but // everyone uses this convention. diff --git a/libgo/go/syscall/socket_linux.go b/libgo/go/syscall/socket_linux.go index 20a37f7..5c0c645 100644 --- a/libgo/go/syscall/socket_linux.go +++ b/libgo/go/syscall/socket_linux.go @@ -1,4 +1,4 @@ -// socket_linux.go -- Socket handling specific to Linux. +// socket_linux.go -- Socket handling specific to GNU/Linux. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -98,7 +98,7 @@ func (sa *RawSockaddrUnix) getLen() (int, int) { } // Assume path ends at NUL. - // This is not technically the Linux semantics for + // This is not technically the GNU/Linux semantics for // abstract Unix domain sockets--they are supposed // to be uninterpreted fixed-size binary blobs--but // everyone uses this convention. diff --git a/libgo/go/syscall/socket_solaris.go b/libgo/go/syscall/socket_solaris.go index 3767078..0c3e6c9 100644 --- a/libgo/go/syscall/socket_solaris.go +++ b/libgo/go/syscall/socket_solaris.go @@ -53,7 +53,7 @@ func (sa *RawSockaddrUnix) getLen() (int, int) { } // Assume path ends at NUL. - // This is not technically the Linux semantics for + // This is not technically the GNU/Linux semantics for // abstract Unix domain sockets--they are supposed // to be uninterpreted fixed-size binary blobs--but // everyone uses this convention. diff --git a/libgo/go/template/exec.go b/libgo/go/template/exec.go index f1590b3..e7fad72 100644 --- a/libgo/go/template/exec.go +++ b/libgo/go/template/exec.go @@ -511,7 +511,7 @@ func (s *state) validateType(value reflect.Value, typ reflect.Type) reflect.Valu // are much more constrained, so it makes more sense there than here. // Besides, one is almost always all you need. switch { - case value.Kind() == reflect.Ptr && value.Elem().Type().AssignableTo(typ): + case value.Kind() == reflect.Ptr && value.Type().Elem().AssignableTo(typ): value = value.Elem() case reflect.PtrTo(value.Type()).AssignableTo(typ) && value.CanAddr(): value = value.Addr() diff --git a/libgo/go/template/exec_test.go b/libgo/go/template/exec_test.go index 8e1894e..50b0ad2 100644 --- a/libgo/go/template/exec_test.go +++ b/libgo/go/template/exec_test.go @@ -231,7 +231,7 @@ var execTests = []execTest{ {"dot complex", "<{{.}}>", "<(16.2-17i)>", 16.2 - 17i, true}, {"dot string", "<{{.}}>", "<hello>", "hello", true}, {"dot slice", "<{{.}}>", "<[-1 -2 -3]>", []int{-1, -2, -3}, true}, - {"dot map", "<{{.}}>", "<map[two:22 one:11]>", map[string]int{"one": 11, "two": 22}, true}, + {"dot map", "<{{.}}>", "<map[two:22]>", map[string]int{"two": 22}, true}, {"dot struct", "<{{.}}>", "<{7 seven}>", struct { a int b string @@ -493,6 +493,50 @@ func TestExecute(t *testing.T) { testExecute(execTests, nil, t) } +var delimPairs = []string{ + "", "", // default + "{{", "}}", // same as default + "<<", ">>", // distinct + "|", "|", // same + "(日)", "(本)", // peculiar +} + +func TestDelims(t *testing.T) { + const hello = "Hello, world" + var value = struct{ Str string }{hello} + for i := 0; i < len(delimPairs); i += 2 { + text := ".Str" + left := delimPairs[i+0] + trueLeft := left + right := delimPairs[i+1] + trueRight := right + if left == "" { // default case + trueLeft = "{{" + } + if right == "" { // default case + trueRight = "}}" + } + text = trueLeft + text + trueRight + // Now add a comment + text += trueLeft + "/*comment*/" + trueRight + // Now add an action containing a string. + text += trueLeft + `"` + trueLeft + `"` + trueRight + // At this point text looks like `{{.Str}}{{/*comment*/}}{{"{{"}}`. + tmpl, err := New("delims").Delims(left, right).Parse(text) + if err != nil { + t.Fatalf("delim %q text %q parse err %s", left, text, err) + } + var b = new(bytes.Buffer) + err = tmpl.Execute(b, value) + if err != nil { + t.Fatalf("delim %q exec err %s", left, err) + } + if b.String() != hello+trueLeft { + t.Errorf("expected %q got %q", hello+trueLeft, b.String()) + } + } +} + // Check that an error from a method flows back to the top. func TestExecuteError(t *testing.T) { b := new(bytes.Buffer) @@ -538,18 +582,19 @@ type Tree struct { Left, Right *Tree } +// Use different delimiters to test Set.Delims. const treeTemplate = ` - {{define "tree"}} + (define "tree") [ - {{.Val}} - {{with .Left}} - {{template "tree" .}} - {{end}} - {{with .Right}} - {{template "tree" .}} - {{end}} + (.Val) + (with .Left) + (template "tree" .) + (end) + (with .Right) + (template "tree" .) + (end) ] - {{end}} + (end) ` func TestTree(t *testing.T) { @@ -590,7 +635,7 @@ func TestTree(t *testing.T) { }, } set := new(Set) - _, err := set.Parse(treeTemplate) + _, err := set.Delims("(", ")").Parse(treeTemplate) if err != nil { t.Fatal("parse error:", err) } diff --git a/libgo/go/template/helper.go b/libgo/go/template/helper.go index c9b0998..1dc90f7 100644 --- a/libgo/go/template/helper.go +++ b/libgo/go/template/helper.go @@ -210,7 +210,8 @@ func ParseTemplateFiles(filenames ...string) (*Set, os.Error) { } // ParseTemplateGlob creates a set by parsing the files matched -// by the pattern, each of which defines a single template. Each +// by the pattern, each of which defines a single template. The pattern +// is processed by filepath.Glob and must match at least one file. Each // template will be named the base name of its file. // Unlike with ParseGlob, each file should be a stand-alone template // definition suitable for Template.Parse (not Set.Parse); that is, the @@ -225,6 +226,9 @@ func ParseTemplateGlob(pattern string) (*Set, os.Error) { if err != nil { return nil, err } + if len(filenames) == 0 { + return nil, fmt.Errorf("pattern matches no files: %#q", pattern) + } for _, filename := range filenames { t, err := ParseFile(filename) if err != nil { diff --git a/libgo/go/template/parse.go b/libgo/go/template/parse.go index b089c59..3068a77 100644 --- a/libgo/go/template/parse.go +++ b/libgo/go/template/parse.go @@ -14,6 +14,8 @@ import ( type Template struct { name string *parse.Tree + leftDelim string + rightDelim string // We use two maps, one for parsing and one for execution. // This separation makes the API cleaner since it doesn't // expose reflection to the client. @@ -38,6 +40,16 @@ func New(name string) *Template { } } +// Delims sets the action delimiters, to be used in a subsequent +// parse, to the specified strings. +// An empty delimiter stands for the corresponding default: {{ or }}. +// The return value is the template, so calls can be chained. +func (t *Template) Delims(left, right string) *Template { + t.leftDelim = left + t.rightDelim = right + return t +} + // Funcs adds the elements of the argument map to the template's function // map. It panics if a value in the map is not a function with appropriate // return type. @@ -51,7 +63,7 @@ func (t *Template) Funcs(funcMap FuncMap) *Template { // Parse parses the template definition string to construct an internal // representation of the template for execution. func (t *Template) Parse(s string) (tmpl *Template, err os.Error) { - t.Tree, err = parse.New(t.name).Parse(s, t.parseFuncs, builtins) + t.Tree, err = parse.New(t.name).Parse(s, t.leftDelim, t.rightDelim, t.parseFuncs, builtins) if err != nil { return nil, err } @@ -67,7 +79,7 @@ func (t *Template) ParseInSet(s string, set *Set) (tmpl *Template, err os.Error) if set != nil { setFuncs = set.parseFuncs } - t.Tree, err = parse.New(t.name).Parse(s, t.parseFuncs, setFuncs, builtins) + t.Tree, err = parse.New(t.name).Parse(s, t.leftDelim, t.rightDelim, t.parseFuncs, setFuncs, builtins) if err != nil { return nil, err } diff --git a/libgo/go/template/parse/lex.go b/libgo/go/template/parse/lex.go index 83ad6c6..16ff590 100644 --- a/libgo/go/template/parse/lex.go +++ b/libgo/go/template/parse/lex.go @@ -119,13 +119,15 @@ type stateFn func(*lexer) stateFn // lexer holds the state of the scanner. type lexer struct { - name string // the name of the input; used only for error reports. - input string // the string being scanned. - state stateFn // the next lexing function to enter - pos int // current position in the input. - start int // start position of this item. - width int // width of last rune read from input. - items chan item // channel of scanned items. + name string // the name of the input; used only for error reports. + input string // the string being scanned. + leftDelim string // start of action. + rightDelim string // end of action. + state stateFn // the next lexing function to enter. + pos int // current position in the input. + start int // start position of this item. + width int // width of last rune read from input. + items chan item // channel of scanned items. } // next returns the next rune in the input. @@ -205,12 +207,20 @@ func (l *lexer) nextItem() item { } // lex creates a new scanner for the input string. -func lex(name, input string) *lexer { +func lex(name, input, left, right string) *lexer { + if left == "" { + left = leftDelim + } + if right == "" { + right = rightDelim + } l := &lexer{ - name: name, - input: input, - state: lexText, - items: make(chan item, 2), // Two items of buffering is sufficient for all state functions + name: name, + input: input, + leftDelim: left, + rightDelim: right, + state: lexText, + items: make(chan item, 2), // Two items of buffering is sufficient for all state functions } return l } @@ -220,14 +230,14 @@ func lex(name, input string) *lexer { const ( leftDelim = "{{" rightDelim = "}}" - leftComment = "{{/*" - rightComment = "*/}}" + leftComment = "/*" + rightComment = "*/" ) // lexText scans until an opening action delimiter, "{{". func lexText(l *lexer) stateFn { for { - if strings.HasPrefix(l.input[l.pos:], leftDelim) { + if strings.HasPrefix(l.input[l.pos:], l.leftDelim) { if l.pos > l.start { l.emit(itemText) } @@ -247,28 +257,28 @@ func lexText(l *lexer) stateFn { // lexLeftDelim scans the left delimiter, which is known to be present. func lexLeftDelim(l *lexer) stateFn { - if strings.HasPrefix(l.input[l.pos:], leftComment) { + if strings.HasPrefix(l.input[l.pos:], l.leftDelim+leftComment) { return lexComment } - l.pos += len(leftDelim) + l.pos += len(l.leftDelim) l.emit(itemLeftDelim) return lexInsideAction } // lexComment scans a comment. The left comment marker is known to be present. func lexComment(l *lexer) stateFn { - i := strings.Index(l.input[l.pos:], rightComment) + i := strings.Index(l.input[l.pos:], rightComment+l.rightDelim) if i < 0 { return l.errorf("unclosed comment") } - l.pos += i + len(rightComment) + l.pos += i + len(rightComment) + len(l.rightDelim) l.ignore() return lexText } // lexRightDelim scans the right delimiter, which is known to be present. func lexRightDelim(l *lexer) stateFn { - l.pos += len(rightDelim) + l.pos += len(l.rightDelim) l.emit(itemRightDelim) return lexText } @@ -278,7 +288,7 @@ func lexInsideAction(l *lexer) stateFn { // Either number, quoted string, or identifier. // Spaces separate and are ignored. // Pipe symbols separate and are emitted. - if strings.HasPrefix(l.input[l.pos:], rightDelim) { + if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { return lexRightDelim } switch r := l.next(); { diff --git a/libgo/go/template/parse/lex_test.go b/libgo/go/template/parse/lex_test.go index d71c8e6..6ee1b47 100644 --- a/libgo/go/template/parse/lex_test.go +++ b/libgo/go/template/parse/lex_test.go @@ -201,8 +201,8 @@ var lexTests = []lexTest{ } // collect gathers the emitted items into a slice. -func collect(t *lexTest) (items []item) { - l := lex(t.name, t.input) +func collect(t *lexTest, left, right string) (items []item) { + l := lex(t.name, t.input, left, right) for { item := l.nextItem() items = append(items, item) @@ -215,7 +215,41 @@ func collect(t *lexTest) (items []item) { func TestLex(t *testing.T) { for _, test := range lexTests { - items := collect(&test) + items := collect(&test, "", "") + if !reflect.DeepEqual(items, test.items) { + t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) + } + } +} + +// Some easy cases from above, but with delimiters $$ and @@ +var lexDelimTests = []lexTest{ + {"punctuation", "$$,@%{{}}@@", []item{ + tLeftDelim, + {itemChar, ","}, + {itemChar, "@"}, + {itemChar, "%"}, + {itemChar, "{"}, + {itemChar, "{"}, + {itemChar, "}"}, + {itemChar, "}"}, + tRightDelim, + tEOF, + }}, + {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}}, + {"for", `$$for @@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}}, + {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}}, + {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}}, +} + +var ( + tLeftDelim = item{itemLeftDelim, "$$"} + tRightDelim = item{itemRightDelim, "@@"} +) + +func TestDelims(t *testing.T) { + for _, test := range lexDelimTests { + items := collect(&test, "$$", "@@") if !reflect.DeepEqual(items, test.items) { t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) } diff --git a/libgo/go/template/parse/node.go b/libgo/go/template/parse/node.go index 6f0b429..7411327 100644 --- a/libgo/go/template/parse/node.go +++ b/libgo/go/template/parse/node.go @@ -390,8 +390,8 @@ func (e *elseNode) String() string { return "{{else}}" } -// IfNode represents an {{if}} action and its commands. -type IfNode struct { +// BranchNode is the common representation of if, range, and with. +type BranchNode struct { NodeType Line int // The line number in the input. Pipe *PipeNode // The pipeline to be evaluated. @@ -399,35 +399,49 @@ type IfNode struct { ElseList *ListNode // What to execute if the value is empty (nil if absent). } -func newIf(line int, pipe *PipeNode, list, elseList *ListNode) *IfNode { - return &IfNode{NodeType: NodeIf, Line: line, Pipe: pipe, List: list, ElseList: elseList} +func (b *BranchNode) String() string { + name := "" + switch b.NodeType { + case NodeIf: + name = "if" + case NodeRange: + name = "range" + case NodeWith: + name = "with" + default: + panic("unknown branch type") + } + if b.ElseList != nil { + return fmt.Sprintf("({{%s %s}} %s {{else}} %s)", name, b.Pipe, b.List, b.ElseList) + } + return fmt.Sprintf("({{%s %s}} %s)", name, b.Pipe, b.List) } -func (i *IfNode) String() string { - if i.ElseList != nil { - return fmt.Sprintf("({{if %s}} %s {{else}} %s)", i.Pipe, i.List, i.ElseList) - } - return fmt.Sprintf("({{if %s}} %s)", i.Pipe, i.List) +// IfNode represents an {{if}} action and its commands. +type IfNode struct { + BranchNode +} + +func newIf(line int, pipe *PipeNode, list, elseList *ListNode) *IfNode { + return &IfNode{BranchNode{NodeType: NodeIf, Line: line, Pipe: pipe, List: list, ElseList: elseList}} } // RangeNode represents a {{range}} action and its commands. type RangeNode struct { - NodeType - Line int // The line number in the input. - Pipe *PipeNode // The pipeline to be evaluated. - List *ListNode // What to execute if the value is non-empty. - ElseList *ListNode // What to execute if the value is empty (nil if absent). + BranchNode } func newRange(line int, pipe *PipeNode, list, elseList *ListNode) *RangeNode { - return &RangeNode{NodeType: NodeRange, Line: line, Pipe: pipe, List: list, ElseList: elseList} + return &RangeNode{BranchNode{NodeType: NodeRange, Line: line, Pipe: pipe, List: list, ElseList: elseList}} } -func (r *RangeNode) String() string { - if r.ElseList != nil { - return fmt.Sprintf("({{range %s}} %s {{else}} %s)", r.Pipe, r.List, r.ElseList) - } - return fmt.Sprintf("({{range %s}} %s)", r.Pipe, r.List) +// WithNode represents a {{with}} action and its commands. +type WithNode struct { + BranchNode +} + +func newWith(line int, pipe *PipeNode, list, elseList *ListNode) *WithNode { + return &WithNode{BranchNode{NodeType: NodeWith, Line: line, Pipe: pipe, List: list, ElseList: elseList}} } // TemplateNode represents a {{template}} action. @@ -448,23 +462,3 @@ func (t *TemplateNode) String() string { } return fmt.Sprintf("{{template %q %s}}", t.Name, t.Pipe) } - -// WithNode represents a {{with}} action and its commands. -type WithNode struct { - NodeType - Line int // The line number in the input. - Pipe *PipeNode // The pipeline to be evaluated. - List *ListNode // What to execute if the value is non-empty. - ElseList *ListNode // What to execute if the value is empty (nil if absent). -} - -func newWith(line int, pipe *PipeNode, list, elseList *ListNode) *WithNode { - return &WithNode{NodeType: NodeWith, Line: line, Pipe: pipe, List: list, ElseList: elseList} -} - -func (w *WithNode) String() string { - if w.ElseList != nil { - return fmt.Sprintf("({{with %s}} %s {{else}} %s)", w.Pipe, w.List, w.ElseList) - } - return fmt.Sprintf("({{with %s}} %s)", w.Pipe, w.List) -} diff --git a/libgo/go/template/parse/parse.go b/libgo/go/template/parse/parse.go index 6918074..9934d82 100644 --- a/libgo/go/template/parse/parse.go +++ b/libgo/go/template/parse/parse.go @@ -145,10 +145,11 @@ func (t *Tree) atEOF() bool { } // Parse parses the template definition string to construct an internal -// representation of the template for execution. -func (t *Tree) Parse(s string, funcs ...map[string]interface{}) (tree *Tree, err os.Error) { +// representation of the template for execution. If either action delimiter +// string is empty, the default ("{{" or "}}") is used. +func (t *Tree) Parse(s, leftDelim, rightDelim string, funcs ...map[string]interface{}) (tree *Tree, err os.Error) { defer t.recover(&err) - t.startParse(funcs, lex(t.Name, s)) + t.startParse(funcs, lex(t.Name, s, leftDelim, rightDelim)) t.parse(true) t.stopParse() return t, nil diff --git a/libgo/go/template/parse/parse_test.go b/libgo/go/template/parse/parse_test.go index 1928c31..f05f6e3 100644 --- a/libgo/go/template/parse/parse_test.go +++ b/libgo/go/template/parse/parse_test.go @@ -236,7 +236,7 @@ var builtins = map[string]interface{}{ func TestParse(t *testing.T) { for _, test := range parseTests { - tmpl, err := New(test.name).Parse(test.input, builtins) + tmpl, err := New(test.name).Parse(test.input, "", "", builtins) switch { case err == nil && !test.ok: t.Errorf("%q: expected error; got none", test.name) diff --git a/libgo/go/template/parse/set.go b/libgo/go/template/parse/set.go index dca41ea..b909f71 100644 --- a/libgo/go/template/parse/set.go +++ b/libgo/go/template/parse/set.go @@ -13,10 +13,10 @@ import ( // Set returns a slice of Trees created by parsing the template set // definition in the argument string. If an error is encountered, // parsing stops and an empty slice is returned with the error. -func Set(text string, funcs ...map[string]interface{}) (tree map[string]*Tree, err os.Error) { +func Set(text, leftDelim, rightDelim string, funcs ...map[string]interface{}) (tree map[string]*Tree, err os.Error) { tree = make(map[string]*Tree) defer (*Tree)(nil).recover(&err) - lex := lex("set", text) + lex := lex("set", text, leftDelim, rightDelim) const context = "define clause" for { t := New("set") // name will be updated once we know it. diff --git a/libgo/go/template/set.go b/libgo/go/template/set.go index f778fd1..712961b 100644 --- a/libgo/go/template/set.go +++ b/libgo/go/template/set.go @@ -17,6 +17,8 @@ import ( // A template may be a member of multiple sets. type Set struct { tmpl map[string]*Template + leftDelim string + rightDelim string parseFuncs FuncMap execFuncs map[string]reflect.Value } @@ -29,6 +31,16 @@ func (s *Set) init() { } } +// Delims sets the action delimiters, to be used in a subsequent +// parse, to the specified strings. +// An empty delimiter stands for the corresponding default: {{ or }}. +// The return value is the set, so calls can be chained. +func (s *Set) Delims(left, right string) *Set { + s.leftDelim = left + s.rightDelim = right + return s +} + // Funcs adds the elements of the argument map to the set's function map. It // panics if a value in the map is not a function with appropriate return // type. @@ -93,7 +105,7 @@ func (s *Set) Execute(wr io.Writer, name string, data interface{}) os.Error { // to the set. If a template is redefined, the element in the set is // overwritten with the new definition. func (s *Set) Parse(text string) (*Set, os.Error) { - trees, err := parse.Set(text, s.parseFuncs, builtins) + trees, err := parse.Set(text, s.leftDelim, s.rightDelim, s.parseFuncs, builtins) if err != nil { return nil, err } diff --git a/libgo/go/testing/example.go b/libgo/go/testing/example.go new file mode 100644 index 0000000..f148951 --- /dev/null +++ b/libgo/go/testing/example.go @@ -0,0 +1,84 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "bytes" + "fmt" + "io" + "os" + "time" +) + +type InternalExample struct { + Name string + F func() + Output string +} + +func RunExamples(examples []InternalExample) (ok bool) { + ok = true + + stdout, stderr := os.Stdout, os.Stderr + defer func() { + os.Stdout, os.Stderr = stdout, stderr + if e := recover(); e != nil { + if err, ok := e.(os.Error); ok { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + panic(e) + } + }() + + for _, eg := range examples { + if *chatty { + fmt.Fprintln(os.Stderr, "=== RUN:", eg.Name) + } + + // capture stdout and stderr for testing purposes + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Stdout, os.Stderr = w, w + outC := make(chan string) + go func() { + buf := new(bytes.Buffer) + _, err := io.Copy(buf, r) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + outC <- buf.String() + }() + + // run example + ns := -time.Nanoseconds() + eg.F() + ns += time.Nanoseconds() + + // close pipe, restore stdout/stderr, get output + w.Close() + os.Stdout, os.Stderr = stdout, stderr + out := <-outC + + // report any errors + if out != eg.Output { + fmt.Fprintf( + os.Stderr, + "--- FAIL: %s\ngot:\n%s\nwant:\n%s\n", + eg.Name, out, eg.Output, + ) + ok = false + } else if *chatty { + tstr := fmt.Sprintf("(%.2f seconds)", float64(ns)/1e9) + fmt.Fprintln(os.Stderr, "--- PASS:", eg.Name, tstr) + } + } + + return +} diff --git a/libgo/go/testing/testing.go b/libgo/go/testing/testing.go index ec4a453..a555cb4 100644 --- a/libgo/go/testing/testing.go +++ b/libgo/go/testing/testing.go @@ -44,8 +44,8 @@ import ( "os" "runtime" "runtime/pprof" - "strings" "strconv" + "strings" "time" ) @@ -65,6 +65,7 @@ var ( cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution") timeout = flag.Int64("test.timeout", 0, "if > 0, sets time limit for tests in seconds") cpuListStr = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test") + parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism") cpuList []int ) @@ -92,9 +93,12 @@ func tabify(s string) string { // T is a type passed to Test functions to manage test state and support formatted test logs. // Logs are accumulated during execution and dumped to standard error when done. type T struct { - errors string - failed bool - ch chan *T + name string // Name of test. + errors string // Error string from test. + failed bool // Test has failed. + ch chan *T // Output for serial tests. + startParallel chan bool // Parallel tests will wait on this. + ns int64 // Duration of test in nanoseconds. } // Fail marks the Test function as having failed but continues execution. @@ -106,6 +110,7 @@ func (t *T) Failed() bool { return t.failed } // FailNow marks the Test function as having failed and stops its execution. // Execution will continue at the next Test. func (t *T) FailNow() { + t.ns = time.Nanoseconds() - t.ns t.Fail() t.ch <- t runtime.Goexit() @@ -145,6 +150,13 @@ func (t *T) Fatalf(format string, args ...interface{}) { t.FailNow() } +// Parallel signals that this test is to be run in parallel with (and only with) +// other parallel tests in this CPU group. +func (t *T) Parallel() { + t.ch <- nil // Release main testing loop + <-t.startParallel // Wait for serial tests to finish +} + // An internal type but exported because it is cross-package; part of the implementation // of gotest. type InternalTest struct { @@ -153,73 +165,97 @@ type InternalTest struct { } func tRunner(t *T, test *InternalTest) { + t.ns = time.Nanoseconds() test.F(t) + t.ns = time.Nanoseconds() - t.ns t.ch <- t } // An internal function but exported because it is cross-package; part of the implementation // of gotest. -func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTest, benchmarks []InternalBenchmark) { +func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { flag.Parse() parseCpuList() before() startAlarm() - RunTests(matchString, tests) + testOk := RunTests(matchString, tests) + exampleOk := RunExamples(examples) + if !testOk || !exampleOk { + fmt.Fprintln(os.Stderr, "FAIL") + os.Exit(1) + } + fmt.Fprintln(os.Stderr, "PASS") stopAlarm() RunBenchmarks(matchString, benchmarks) after() } -func RunTests(matchString func(pat, str string) (bool, os.Error), tests []InternalTest) { - ok := true +func report(t *T) { + tstr := fmt.Sprintf("(%.2f seconds)", float64(t.ns)/1e9) + format := "--- %s: %s %s\n%s" + if t.failed { + fmt.Fprintf(os.Stderr, format, "FAIL", t.name, tstr, t.errors) + } else if *chatty { + fmt.Fprintf(os.Stderr, format, "PASS", t.name, tstr, t.errors) + } +} + +func RunTests(matchString func(pat, str string) (bool, os.Error), tests []InternalTest) (ok bool) { + ok = true if len(tests) == 0 { - println("testing: warning: no tests to run") + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + return } - for i := 0; i < len(tests); i++ { - matched, err := matchString(*match, tests[i].Name) - if err != nil { - println("invalid regexp for -test.run:", err.String()) - os.Exit(1) - } - if !matched { - continue - } - for _, procs := range cpuList { - runtime.GOMAXPROCS(procs) + ch := make(chan *T) + for _, procs := range cpuList { + runtime.GOMAXPROCS(procs) + + numParallel := 0 + startParallel := make(chan bool) + + for i := 0; i < len(tests); i++ { + matched, err := matchString(*match, tests[i].Name) + if err != nil { + println("invalid regexp for -test.run:", err.String()) + os.Exit(1) + } + if !matched { + continue + } testName := tests[i].Name if procs != 1 { testName = fmt.Sprintf("%s-%d", tests[i].Name, procs) } + t := &T{ch: ch, name: testName, startParallel: startParallel} if *chatty { - println("=== RUN ", testName) + println("=== RUN", t.name) } - ns := -time.Nanoseconds() - t := new(T) - t.ch = make(chan *T) go tRunner(t, &tests[i]) - <-t.ch - ns += time.Nanoseconds() - tstr := fmt.Sprintf("(%.2f seconds)", float64(ns)/1e9) - if p := runtime.GOMAXPROCS(-1); t.failed == false && p != procs { - t.failed = true - t.errors = fmt.Sprintf("%s left GOMAXPROCS set to %d\n", testName, p) + out := <-t.ch + if out == nil { // Parallel run. + numParallel++ + continue } - if t.failed { - println("--- FAIL:", testName, tstr) - print(t.errors) - ok = false - } else if *chatty { - println("--- PASS:", testName, tstr) - print(t.errors) + report(t) + ok = ok && !out.failed + } + + running := 0 + for numParallel+running > 0 { + if running < *parallel && numParallel > 0 { + startParallel <- true + running++ + numParallel-- + continue } + t := <-ch + report(t) + ok = ok && !t.failed + running-- } } - if !ok { - println("FAIL") - os.Exit(1) - } - println("PASS") + return } // before runs before all testing. diff --git a/libgo/go/time/format.go b/libgo/go/time/format.go index 5ddd548..50e96a5 100644 --- a/libgo/go/time/format.go +++ b/libgo/go/time/format.go @@ -232,9 +232,27 @@ var longMonthNames = []string{ "December", } +// match returns true if s1 and s2 match ignoring case. +// It is assumed s1 and s2 are the same length. +func match(s1, s2 string) bool { + for i := 0; i < len(s1); i++ { + c1 := s1[i] + c2 := s2[i] + if c1 != c2 { + // Switch to lower-case; 'a'-'A' is known to be a single bit. + c1 |= 'a' - 'A' + c2 |= 'a' - 'A' + if c1 != c2 || c1 < 'a' || c1 > 'z' { + return false + } + } + } + return true +} + func lookup(tab []string, val string) (int, string, os.Error) { for i, v := range tab { - if len(val) >= len(v) && val[0:len(v)] == v { + if len(val) >= len(v) && match(val[0:len(v)], v) { return i, val[len(v):], nil } } @@ -296,9 +314,9 @@ func (t *Time) Format(layout string) string { case stdZeroMonth: p = zeroPad(t.Month) case stdWeekDay: - p = shortDayNames[t.Weekday] + p = shortDayNames[t.Weekday()] case stdLongWeekDay: - p = longDayNames[t.Weekday] + p = longDayNames[t.Weekday()] case stdDay: p = strconv.Itoa(t.Day) case stdUnderDay: @@ -485,7 +503,8 @@ func skip(value, prefix string) (string, os.Error) { // (such as having the wrong day of the week), the returned value will also // be inconsistent. In any case, the elements of the returned time will be // sane: hours in 0..23, minutes in 0..59, day of month in 1..31, etc. -// Years must be in the range 0000..9999. +// Years must be in the range 0000..9999. The day of the week is checked +// for syntax but it is otherwise ignored. func Parse(alayout, avalue string) (*Time, os.Error) { var t Time rangeErrString := "" // set if a value is out of range @@ -538,9 +557,10 @@ func Parse(alayout, avalue string) (*Time, os.Error) { rangeErrString = "month" } case stdWeekDay: - t.Weekday, value, err = lookup(shortDayNames, value) + // Ignore weekday except for error checking. + _, value, err = lookup(shortDayNames, value) case stdLongWeekDay: - t.Weekday, value, err = lookup(longDayNames, value) + _, value, err = lookup(longDayNames, value) case stdDay, stdUnderDay, stdZeroDay: if std == stdUnderDay && len(value) > 0 && value[0] == ' ' { value = value[1:] diff --git a/libgo/go/time/internal_test.go b/libgo/go/time/internal_test.go new file mode 100644 index 0000000..d7e7076 --- /dev/null +++ b/libgo/go/time/internal_test.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time + +func init() { + // force US/Pacific for time zone tests + onceSetupZone.Do(setupTestingZone) +} + +var Interrupt = interrupt diff --git a/libgo/go/time/sleep_test.go b/libgo/go/time/sleep_test.go index 9b59767..9d16c52 100644 --- a/libgo/go/time/sleep_test.go +++ b/libgo/go/time/sleep_test.go @@ -7,7 +7,6 @@ package time_test import ( "fmt" "os" - "syscall" "testing" "sort" . "time" @@ -17,7 +16,7 @@ func TestSleep(t *testing.T) { const delay = int64(100e6) go func() { Sleep(delay / 2) - syscall.Kill(os.Getpid(), syscall.SIGCHLD) + Interrupt() }() start := Nanoseconds() Sleep(delay) diff --git a/libgo/go/time/sys_plan9.go b/libgo/go/time/sys_plan9.go index abe8649..9ae0161 100644 --- a/libgo/go/time/sys_plan9.go +++ b/libgo/go/time/sys_plan9.go @@ -16,3 +16,8 @@ func sysSleep(t int64) os.Error { } return nil } + +// for testing: whatever interrupts a sleep +func interrupt() { + // cannot predict pid, don't want to kill group +} diff --git a/libgo/go/time/sys_unix.go b/libgo/go/time/sys_unix.go new file mode 100644 index 0000000..0119bdf --- /dev/null +++ b/libgo/go/time/sys_unix.go @@ -0,0 +1,25 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux openbsd + +package time + +import ( + "os" + "syscall" +) + +func sysSleep(t int64) os.Error { + errno := syscall.Sleep(t) + if errno != 0 && errno != syscall.EINTR { + return os.NewSyscallError("sleep", errno) + } + return nil +} + +// for testing: whatever interrupts a sleep +func interrupt() { + syscall.Kill(os.Getpid(), syscall.SIGCHLD) +} diff --git a/libgo/go/time/sys_posix.go b/libgo/go/time/sys_windows.go index 0d1eb72..feff90b 100644 --- a/libgo/go/time/sys_posix.go +++ b/libgo/go/time/sys_windows.go @@ -16,3 +16,7 @@ func sysSleep(t int64) os.Error { } return nil } + +// for testing: whatever interrupts a sleep +func interrupt() { +} diff --git a/libgo/go/time/time.go b/libgo/go/time/time.go index 0e05da4..859b316 100644 --- a/libgo/go/time/time.go +++ b/libgo/go/time/time.go @@ -22,7 +22,6 @@ type Time struct { Month, Day int // Jan-2 is 1, 2 Hour, Minute, Second int // 15:04:05 is 15, 4, 5. Nanosecond int // Fractional second. - Weekday int // Sunday, Monday, ... ZoneOffset int // seconds east of UTC, e.g. -7*60*60 for -0700 Zone string // e.g., "MST" } @@ -63,12 +62,6 @@ func SecondsToUTC(sec int64) *Time { t.Minute = int((sec / 60) % 60) t.Second = int(sec % 60) - // Day 0 = January 1, 1970 was a Thursday - t.Weekday = int((day + Thursday) % 7) - if t.Weekday < 0 { - t.Weekday += 7 - } - // Change day from 0 = 1970 to 0 = 2001, // to make leap year calculations easier // (2001 begins 4-, 100-, and 400-year cycles ending in a leap year.) @@ -228,3 +221,19 @@ func (t *Time) Seconds() int64 { func (t *Time) Nanoseconds() int64 { return t.Seconds()*1e9 + int64(t.Nanosecond) } + +// Weekday returns the time's day of the week. Sunday is day 0. +func (t *Time) Weekday() int { + sec := t.Seconds() + int64(t.ZoneOffset) + day := sec / secondsPerDay + sec -= day * secondsPerDay + if sec < 0 { + day-- + } + // Day 0 = January 1, 1970 was a Thursday + weekday := int((day + Thursday) % 7) + if weekday < 0 { + weekday += 7 + } + return weekday +} diff --git a/libgo/go/time/time_test.go b/libgo/go/time/time_test.go index dceed49..353976c 100644 --- a/libgo/go/time/time_test.go +++ b/libgo/go/time/time_test.go @@ -5,7 +5,6 @@ package time_test import ( - "os" "strconv" "strings" "testing" @@ -13,13 +12,6 @@ import ( . "time" ) -func init() { - // Force US Pacific time for daylight-savings - // tests below (localtests). Needs to be set - // before the first call into the time library. - os.Setenv("TZ", "America/Los_Angeles") -} - // We should be in PST/PDT, but if the time zone files are missing we // won't be. The purpose of this test is to at least explain why some of // the subsequent tests fail. @@ -38,31 +30,31 @@ type TimeTest struct { } var utctests = []TimeTest{ - {0, Time{1970, 1, 1, 0, 0, 0, 0, Thursday, 0, "UTC"}}, - {1221681866, Time{2008, 9, 17, 20, 4, 26, 0, Wednesday, 0, "UTC"}}, - {-1221681866, Time{1931, 4, 16, 3, 55, 34, 0, Thursday, 0, "UTC"}}, - {-11644473600, Time{1601, 1, 1, 0, 0, 0, 0, Monday, 0, "UTC"}}, - {599529660, Time{1988, 12, 31, 0, 1, 0, 0, Saturday, 0, "UTC"}}, - {978220860, Time{2000, 12, 31, 0, 1, 0, 0, Sunday, 0, "UTC"}}, - {1e18, Time{31688740476, 10, 23, 1, 46, 40, 0, Friday, 0, "UTC"}}, - {-1e18, Time{-31688736537, 3, 10, 22, 13, 20, 0, Tuesday, 0, "UTC"}}, - {0x7fffffffffffffff, Time{292277026596, 12, 4, 15, 30, 7, 0, Sunday, 0, "UTC"}}, - {-0x8000000000000000, Time{-292277022657, 1, 27, 8, 29, 52, 0, Sunday, 0, "UTC"}}, + {0, Time{1970, 1, 1, 0, 0, 0, 0, 0, "UTC"}}, + {1221681866, Time{2008, 9, 17, 20, 4, 26, 0, 0, "UTC"}}, + {-1221681866, Time{1931, 4, 16, 3, 55, 34, 0, 0, "UTC"}}, + {-11644473600, Time{1601, 1, 1, 0, 0, 0, 0, 0, "UTC"}}, + {599529660, Time{1988, 12, 31, 0, 1, 0, 0, 0, "UTC"}}, + {978220860, Time{2000, 12, 31, 0, 1, 0, 0, 0, "UTC"}}, + {1e18, Time{31688740476, 10, 23, 1, 46, 40, 0, 0, "UTC"}}, + {-1e18, Time{-31688736537, 3, 10, 22, 13, 20, 0, 0, "UTC"}}, + {0x7fffffffffffffff, Time{292277026596, 12, 4, 15, 30, 7, 0, 0, "UTC"}}, + {-0x8000000000000000, Time{-292277022657, 1, 27, 8, 29, 52, 0, 0, "UTC"}}, } var nanoutctests = []TimeTest{ - {0, Time{1970, 1, 1, 0, 0, 0, 1e8, Thursday, 0, "UTC"}}, - {1221681866, Time{2008, 9, 17, 20, 4, 26, 2e8, Wednesday, 0, "UTC"}}, + {0, Time{1970, 1, 1, 0, 0, 0, 1e8, 0, "UTC"}}, + {1221681866, Time{2008, 9, 17, 20, 4, 26, 2e8, 0, "UTC"}}, } var localtests = []TimeTest{ - {0, Time{1969, 12, 31, 16, 0, 0, 0, Wednesday, -8 * 60 * 60, "PST"}}, - {1221681866, Time{2008, 9, 17, 13, 4, 26, 0, Wednesday, -7 * 60 * 60, "PDT"}}, + {0, Time{1969, 12, 31, 16, 0, 0, 0, -8 * 60 * 60, "PST"}}, + {1221681866, Time{2008, 9, 17, 13, 4, 26, 0, -7 * 60 * 60, "PDT"}}, } var nanolocaltests = []TimeTest{ - {0, Time{1969, 12, 31, 16, 0, 0, 1e8, Wednesday, -8 * 60 * 60, "PST"}}, - {1221681866, Time{2008, 9, 17, 13, 4, 26, 3e8, Wednesday, -7 * 60 * 60, "PDT"}}, + {0, Time{1969, 12, 31, 16, 0, 0, 1e8, -8 * 60 * 60, "PST"}}, + {1221681866, Time{2008, 9, 17, 13, 4, 26, 3e8, -7 * 60 * 60, "PDT"}}, } func same(t, u *Time) bool { @@ -73,7 +65,7 @@ func same(t, u *Time) bool { t.Minute == u.Minute && t.Second == u.Second && t.Nanosecond == u.Nanosecond && - t.Weekday == u.Weekday && + t.Weekday() == u.Weekday() && t.ZoneOffset == u.ZoneOffset && t.Zone == u.Zone } @@ -181,9 +173,9 @@ type TimeFormatTest struct { } var rfc3339Formats = []TimeFormatTest{ - {Time{2008, 9, 17, 20, 4, 26, 0, Wednesday, 0, "UTC"}, "2008-09-17T20:04:26Z"}, - {Time{1994, 9, 17, 20, 4, 26, 0, Wednesday, -18000, "EST"}, "1994-09-17T20:04:26-05:00"}, - {Time{2000, 12, 26, 1, 15, 6, 0, Wednesday, 15600, "OTO"}, "2000-12-26T01:15:06+04:20"}, + {Time{2008, 9, 17, 20, 4, 26, 0, 0, "UTC"}, "2008-09-17T20:04:26Z"}, + {Time{1994, 9, 17, 20, 4, 26, 0, -18000, "EST"}, "1994-09-17T20:04:26-05:00"}, + {Time{2000, 12, 26, 1, 15, 6, 0, 15600, "OTO"}, "2000-12-26T01:15:06+04:20"}, } func TestRFC3339Conversion(t *testing.T) { @@ -260,6 +252,9 @@ var parseTests = []ParseTest{ // Amount of white space should not matter. {"ANSIC", ANSIC, "Thu Feb 4 21:00:57 2010", false, true, 1, 0}, {"ANSIC", ANSIC, "Thu Feb 4 21:00:57 2010", false, true, 1, 0}, + // Case should not matter + {"ANSIC", ANSIC, "THU FEB 4 21:00:57 2010", false, true, 1, 0}, + {"ANSIC", ANSIC, "thu feb 4 21:00:57 2010", false, true, 1, 0}, // Fractional seconds. {"millisecond", "Mon Jan _2 15:04:05.000 2006", "Thu Feb 4 21:00:57.012 2010", false, true, 1, 3}, {"microsecond", "Mon Jan _2 15:04:05.000000 2006", "Thu Feb 4 21:00:57.012345 2010", false, true, 1, 6}, @@ -331,8 +326,8 @@ func checkTime(time *Time, test *ParseTest, t *testing.T) { if test.hasTZ && time.ZoneOffset != -28800 { t.Errorf("%s: bad tz offset: %d not %d", test.name, time.ZoneOffset, -28800) } - if test.hasWD && time.Weekday != 4 { - t.Errorf("%s: bad weekday: %d not %d", test.name, time.Weekday, 4) + if test.hasWD && time.Weekday() != 4 { + t.Errorf("%s: bad weekday: %d not %d", test.name, time.Weekday(), 4) } } @@ -458,11 +453,11 @@ func Test12AMIsMidnight(t *testing.T) { // Check that a time without a Zone still produces a (numeric) time zone // when formatted with MST as a requested zone. func TestMissingZone(t *testing.T) { - time, err := Parse(RubyDate, "Tue Feb 02 16:10:03 -0500 2006") + time, err := Parse(RubyDate, "Thu Feb 02 16:10:03 -0500 2006") if err != nil { t.Fatal("error parsing date:", err) } - expect := "Tue Feb 2 16:10:03 -0500 2006" // -0500 not EST + expect := "Thu Feb 2 16:10:03 -0500 2006" // -0500 not EST str := time.Format(UnixDate) // uses MST as its time zone if str != expect { t.Errorf("expected %q got %q", expect, str) diff --git a/libgo/go/time/zoneinfo_plan9.go b/libgo/go/time/zoneinfo_plan9.go index 3c3e7c4..577ef85 100644 --- a/libgo/go/time/zoneinfo_plan9.go +++ b/libgo/go/time/zoneinfo_plan9.go @@ -57,3 +57,19 @@ func setupZone() { } zones = parseZones(t) } + +func setupTestingZone() { + f, err := os.Open("/adm/timezone/US_Pacific") + if err != nil { + return + } + defer f.Close() + l, _ := f.Seek(0, 2) + f.Seek(0, 0) + buf := make([]byte, l) + _, err = f.Read(buf) + if err != nil { + return + } + zones = parseZones(string(buf)) +} diff --git a/libgo/go/time/zoneinfo_posix.go b/libgo/go/time/zoneinfo_posix.go index b492164..b0fa6c3 100644 --- a/libgo/go/time/zoneinfo_posix.go +++ b/libgo/go/time/zoneinfo_posix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd plan9 + package time import "sync" diff --git a/libgo/go/time/zoneinfo_unix.go b/libgo/go/time/zoneinfo_unix.go index f3ea7b6..0dc4235 100644 --- a/libgo/go/time/zoneinfo_unix.go +++ b/libgo/go/time/zoneinfo_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build darwin freebsd linux openbsd + // Parse "zoneinfo" time zone file. // This is a fairly standard file format used on OS X, Linux, BSD, Sun, and others. // See tzfile(5), http://en.wikipedia.org/wiki/Zoneinfo, @@ -185,6 +187,11 @@ func readinfofile(name string) ([]zonetime, bool) { return parseinfo(buf) } +func setupTestingZone() { + os.Setenv("TZ", "America/Los_Angeles") + setupZone() +} + func setupZone() { // consult $TZ to find the time zone to use. // no $TZ means use the system default /etc/localtime. diff --git a/libgo/go/time/zoneinfo_windows.go b/libgo/go/time/zoneinfo_windows.go index fabc006..41c4819 100644 --- a/libgo/go/time/zoneinfo_windows.go +++ b/libgo/go/time/zoneinfo_windows.go @@ -27,9 +27,30 @@ type zone struct { prev *zone } +// BUG(rsc): On Windows, time zone abbreviations are unavailable. +// This package constructs them using the capital letters from a longer +// time zone description. + // Populate zone struct with Windows supplied information. Returns true, if data is valid. func (z *zone) populate(bias, biasdelta int32, d *syscall.Systemtime, name []uint16) (dateisgood bool) { - z.name = syscall.UTF16ToString(name) + // name is 'Pacific Standard Time' but we want 'PST'. + // Extract just capital letters. It's not perfect but the + // information we need is not available from the kernel. + // Because time zone abbreviations are not unique, + // Windows refuses to expose them. + // + // http://social.msdn.microsoft.com/Forums/eu/vclanguage/thread/a87e1d25-fb71-4fe0-ae9c-a9578c9753eb + // http://stackoverflow.com/questions/4195948/windows-time-zone-abbreviations-in-asp-net + short := make([]uint16, len(name)) + w := 0 + for _, c := range name { + if 'A' <= c && c <= 'Z' { + short[w] = c + w++ + } + } + z.name = syscall.UTF16ToString(short[:w]) + z.offset = int(bias) z.year = int64(d.Year) z.month = int(d.Month) @@ -49,7 +70,15 @@ func (z *zone) populate(bias, biasdelta int32, d *syscall.Systemtime, name []uin // Pre-calculate cutoff time in seconds since the Unix epoch, if data is supplied in "absolute" format. func (z *zone) preCalculateAbsSec() { if z.year != 0 { - z.abssec = (&Time{z.year, int(z.month), int(z.day), int(z.hour), int(z.minute), int(z.second), 0, 0, 0, ""}).Seconds() + t := &Time{ + Year: z.year, + Month: int(z.month), + Day: int(z.day), + Hour: int(z.hour), + Minute: int(z.minute), + Second: int(z.second), + } + z.abssec = t.Seconds() // Time given is in "local" time. Adjust it for "utc". z.abssec -= int64(z.prev.offset) } @@ -62,9 +91,16 @@ func (z *zone) cutoffSeconds(year int64) int64 { // z.dayofweek is appropriate weekday (Sunday=0 to Saturday=6) // z.day is week within the month (1 to 5, where 5 is last week of the month) // z.hour, z.minute and z.second are absolute time - t := &Time{year, int(z.month), 1, int(z.hour), int(z.minute), int(z.second), 0, 0, 0, ""} + t := &Time{ + Year: year, + Month: int(z.month), + Day: 1, + Hour: int(z.hour), + Minute: int(z.minute), + Second: int(z.second), + } t = SecondsToUTC(t.Seconds()) - i := int(z.dayofweek) - t.Weekday + i := int(z.dayofweek) - t.Weekday() if i < 0 { i += 7 } @@ -129,6 +165,10 @@ func setupZone() { initError = os.NewSyscallError("GetTimeZoneInformation", e) return } + setupZoneFromTZI(&i) +} + +func setupZoneFromTZI(i *syscall.Timezoneinformation) { if !tz.std.populate(i.Bias, i.StandardBias, &i.StandardDate, i.StandardName[0:]) { tz.disabled = true tz.offsetIfDisabled = tz.std.offset @@ -144,6 +184,23 @@ func setupZone() { tz.januaryIsStd = tz.dst.cutoffSeconds(t.Year) < tz.std.cutoffSeconds(t.Year) } +var usPacific = syscall.Timezoneinformation{ + Bias: 8 * 60, + StandardName: [32]uint16{ + 'P', 'a', 'c', 'i', 'f', 'i', 'c', ' ', 'S', 't', 'a', 'n', 'd', 'a', 'r', 'd', ' ', 'T', 'i', 'm', 'e', + }, + StandardDate: syscall.Systemtime{Month: 11, Day: 1, Hour: 2}, + DaylightName: [32]uint16{ + 'P', 'a', 'c', 'i', 'f', 'i', 'c', ' ', 'D', 'a', 'y', 'l', 'i', 'g', 'h', 't', ' ', 'T', 'i', 'm', 'e', + }, + DaylightDate: syscall.Systemtime{Month: 3, Day: 2, Hour: 2}, + DaylightBias: -60, +} + +func setupTestingZone() { + setupZoneFromTZI(&usPacific) +} + // Look up the correct time zone (daylight savings or not) for the given unix time, in the current location. func lookupTimezone(sec int64) (zone string, offset int) { onceSetupZone.Do(setupZone) diff --git a/libgo/go/try/try.go b/libgo/go/try/try.go deleted file mode 100644 index 2a3dbf9..0000000 --- a/libgo/go/try/try.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package try contains the executable part of the gotry command. -// It is not intended for general use. -package try - -import ( - "fmt" - "io" - "os" - "reflect" - "unicode" -) - -var output io.Writer = os.Stdout // redirected when testing - -// Main is called directly from the gotry-generated Go source file to perform -// the evaluations. -func Main(pkg, firstArg string, functions map[string]interface{}, args []interface{}) { - switch len(args) { - case 0: - // Nothing to do. - case 1: - // Compiler has already evaluated the expression; just print the result. - printSlice(firstArg, args) - default: - // See if methods satisfy the expressions. - tryMethods(pkg, firstArg, args) - // See if functions satisfy the expressions. - for name, fn := range functions { - tryFunction(pkg, name, fn, args) - } - } -} - -// printSlice prints the zeroth element of the args slice, which should (by construction) -// itself be a slice of interface{}. -func printSlice(firstArg string, args []interface{}) { - // Args should be length 1 and a slice. - if len(args) != 1 { - return - } - arg, ok := args[0].([]interface{}) - if !ok { - return - } - fmt.Fprintf(output, "%s = ", firstArg) - if len(arg) > 1 { - fmt.Fprint(output, "(") - } - for i, a := range arg { - if i > 0 { - fmt.Fprint(output, ", ") - } - fmt.Fprintf(output, "%#v", a) - } - if len(arg) > 1 { - fmt.Fprint(output, ")") - } - fmt.Fprint(output, "\n") -} - -// tryMethods sees if the zeroth arg has methods, and if so treats them as potential -// functions to satisfy the remaining arguments. -func tryMethods(pkg, firstArg string, args []interface{}) { - defer func() { recover() }() - // Is the first argument something with methods? - v := reflect.ValueOf(args[0]) - typ := v.Type() - if typ.NumMethod() == 0 { - return - } - for i := 0; i < typ.NumMethod(); i++ { - if unicode.IsUpper(int(typ.Method(i).Name[0])) { - tryMethod(pkg, firstArg, typ.Method(i), args) - } - } -} - -// tryMethod converts a method to a function for tryOneFunction. -func tryMethod(pkg, firstArg string, method reflect.Method, args []interface{}) { - rfn := method.Func - typ := method.Type - name := method.Name - tryOneFunction(pkg, firstArg, name, typ, rfn, args) -} - -// tryFunction sees if fn satisfies the arguments. -func tryFunction(pkg, name string, fn interface{}, args []interface{}) { - defer func() { recover() }() - rfn := reflect.ValueOf(fn) - typ := rfn.Type() - tryOneFunction(pkg, "", name, typ, rfn, args) -} - -// tryOneFunction is the common code for tryMethod and tryFunction. -func tryOneFunction(pkg, firstArg, name string, typ reflect.Type, rfn reflect.Value, args []interface{}) { - // Any results? - if typ.NumOut() == 0 { - return // Nothing to do. - } - // Right number of arguments + results? - if typ.NumIn()+typ.NumOut() != len(args) { - return - } - // Right argument and result types? - for i, a := range args { - if i < typ.NumIn() { - if !compatible(a, typ.In(i)) { - return - } - } else { - if !compatible(a, typ.Out(i-typ.NumIn())) { - return - } - } - } - // Build the call args. - argsVal := make([]reflect.Value, typ.NumIn()+typ.NumOut()) - for i, a := range args { - argsVal[i] = reflect.ValueOf(a) - } - // Call the function and see if the results are as expected. - resultVal := rfn.Call(argsVal[:typ.NumIn()]) - for i, v := range resultVal { - if !reflect.DeepEqual(v.Interface(), args[i+typ.NumIn()]) { - return - } - } - // Present the result including a godoc command to get more information. - firstIndex := 0 - if firstArg != "" { - fmt.Fprintf(output, "%s.%s(", firstArg, name) - firstIndex = 1 - } else { - fmt.Fprintf(output, "%s.%s(", pkg, name) - } - for i := firstIndex; i < typ.NumIn(); i++ { - if i > firstIndex { - fmt.Fprint(output, ", ") - } - fmt.Fprintf(output, "%#v", args[i]) - } - fmt.Fprint(output, ") = ") - if typ.NumOut() > 1 { - fmt.Fprint(output, "(") - } - for i := 0; i < typ.NumOut(); i++ { - if i > 0 { - fmt.Fprint(output, ", ") - } - fmt.Fprintf(output, "%#v", resultVal[i].Interface()) - } - if typ.NumOut() > 1 { - fmt.Fprint(output, ")") - } - fmt.Fprintf(output, " // godoc %s %s\n", pkg, name) -} - -// compatible reports whether the argument is compatible with the type. -func compatible(arg interface{}, typ reflect.Type) bool { - if reflect.TypeOf(arg) == typ { - return true - } - if arg == nil { - // nil is OK if the type is an interface. - if typ.Kind() == reflect.Interface { - return true - } - } - return false -} diff --git a/libgo/go/try/try_test.go b/libgo/go/try/try_test.go deleted file mode 100644 index 617b2c7..0000000 --- a/libgo/go/try/try_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package try - -import ( - "bytes" - "regexp" // Used as the package to try. - "testing" -) - -// The global functions in package regexp at time of writing. -// Doesn't need to be updated unless the entries in this list become invalid. -var functions = map[string]interface{}{ - "Compile": regexp.Compile, - "Match": regexp.Match, - "MatchString": regexp.MatchString, - "MustCompile": regexp.MustCompile, - "QuoteMeta": regexp.QuoteMeta, -} - -// A wraps arguments to make the test cases nicer to read. -func A(args ...interface{}) []interface{} { - return args -} - -type Test struct { - firstArg string // only needed if there is exactly one argument - result string // minus final newline; might be just the godoc string - args []interface{} -} - -var testRE = regexp.MustCompile("a(.)(.)d") - -var tests = []Test{ - // A simple expression. The final value is a slice in case the expression is multivalue. - {"3+4", "3+4 = 7", A([]interface{}{7})}, - // A search for a function. - {"", "regexp QuoteMeta", A("([])", `\(\[\]\)`)}, - // A search for a function with multiple return values. - {"", "regexp MatchString", A("abc", "xabcd", true, nil)}, - // Searches for methods. - {"", "regexp MatchString", A(testRE, "xabcde", true)}, - {"", "regexp NumSubexp", A(testRE, 2)}, -} - -func TestAll(t *testing.T) { - re := regexp.MustCompile(".*// godoc ") - for _, test := range tests { - b := new(bytes.Buffer) - output = b - Main("regexp", test.firstArg, functions, test.args) - expect := test.result + "\n" - got := re.ReplaceAllString(b.String(), "") - if got != expect { - t.Errorf("expected %q; got %q", expect, got) - } - } -} diff --git a/libgo/go/unicode/script_test.go b/libgo/go/unicode/script_test.go index b37ad18..dfd636d 100644 --- a/libgo/go/unicode/script_test.go +++ b/libgo/go/unicode/script_test.go @@ -206,7 +206,7 @@ func TestScripts(t *testing.T) { if !Is(Scripts[test.script], test.rune) { t.Errorf("IsScript(%U, %s) = false, want true", test.rune, test.script) } - notTested[test.script] = false, false + delete(notTested, test.script) } for _, test := range outTest { if Is(Scripts[test.script], test.rune) { @@ -230,7 +230,7 @@ func TestCategories(t *testing.T) { if !Is(Categories[test.script], test.rune) { t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script) } - notTested[test.script] = false, false + delete(notTested, test.script) } for k := range notTested { t.Error("category not tested:", k) @@ -249,7 +249,7 @@ func TestProperties(t *testing.T) { if !Is(Properties[test.script], test.rune) { t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script) } - notTested[test.script] = false, false + delete(notTested, test.script) } for k := range notTested { t.Error("property not tested:", k) diff --git a/libgo/go/url/url.go b/libgo/go/url/url.go index d07b016..dd1f93d 100644 --- a/libgo/go/url/url.go +++ b/libgo/go/url/url.go @@ -517,7 +517,7 @@ func (v Values) Add(key, value string) { // Del deletes the values associated with key. func (v Values) Del(key string) { - v[key] = nil, false + delete(v, key) } // ParseQuery parses the URL-encoded query string and returns @@ -532,20 +532,28 @@ func ParseQuery(query string) (m Values, err os.Error) { } func parseQuery(m Values, query string) (err os.Error) { - for _, kv := range strings.Split(query, "&") { - if len(kv) == 0 { + for query != "" { + key := query + if i := strings.IndexAny(key, "&;"); i >= 0 { + key, query = key[:i], key[i+1:] + } else { + query = "" + } + if key == "" { continue } - kvPair := strings.SplitN(kv, "=", 2) - - var key, value string - var e os.Error - key, e = QueryUnescape(kvPair[0]) - if e == nil && len(kvPair) > 1 { - value, e = QueryUnescape(kvPair[1]) + value := "" + if i := strings.Index(key, "="); i >= 0 { + key, value = key[:i], key[i+1:] + } + key, err1 := QueryUnescape(key) + if err1 != nil { + err = err1 + continue } - if e != nil { - err = e + value, err1 = QueryUnescape(value) + if err1 != nil { + err = err1 continue } m[key] = append(m[key], value) diff --git a/libgo/go/url/url_test.go b/libgo/go/url/url_test.go index af394d4..8c27e18e 100644 --- a/libgo/go/url/url_test.go +++ b/libgo/go/url/url_test.go @@ -11,11 +11,6 @@ import ( "testing" ) -// TODO(rsc): -// test Unescape -// test Escape -// test Parse - type URLTest struct { in string out *URL @@ -696,3 +691,60 @@ func TestQueryValues(t *testing.T) { t.Errorf("second Get(bar) = %q, want %q", g, e) } } + +type parseTest struct { + query string + out Values +} + +var parseTests = []parseTest{ + { + query: "a=1&b=2", + out: Values{"a": []string{"1"}, "b": []string{"2"}}, + }, + { + query: "a=1&a=2&a=banana", + out: Values{"a": []string{"1", "2", "banana"}}, + }, + { + query: "ascii=%3Ckey%3A+0x90%3E", + out: Values{"ascii": []string{"<key: 0x90>"}}, + }, + { + query: "a=1;b=2", + out: Values{"a": []string{"1"}, "b": []string{"2"}}, + }, + { + query: "a=1&a=2;a=banana", + out: Values{"a": []string{"1", "2", "banana"}}, + }, +} + +func TestParseQuery(t *testing.T) { + for i, test := range parseTests { + form, err := ParseQuery(test.query) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + if len(form) != len(test.out) { + t.Errorf("test %d: len(form) = %d, want %d", i, len(form), len(test.out)) + } + for k, evs := range test.out { + vs, ok := form[k] + if !ok { + t.Errorf("test %d: Missing key %q", i, k) + continue + } + if len(vs) != len(evs) { + t.Errorf("test %d: len(form[%q]) = %d, want %d", i, k, len(vs), len(evs)) + continue + } + for j, ev := range evs { + if v := vs[j]; v != ev { + t.Errorf("test %d: form[%q][%d] = %q, want %q", i, k, j, v, ev) + } + } + } + } +} diff --git a/libgo/go/utf8/utf8.go b/libgo/go/utf8/utf8.go index 8910e17d..3cd919d 100644 --- a/libgo/go/utf8/utf8.go +++ b/libgo/go/utf8/utf8.go @@ -354,3 +354,40 @@ func RuneCountInString(s string) (n int) { // an encoded rune. Second and subsequent bytes always have the top // two bits set to 10. func RuneStart(b byte) bool { return b&0xC0 != 0x80 } + +// Valid reports whether p consists entirely of valid UTF-8-encoded runes. +func Valid(p []byte) bool { + i := 0 + for i < len(p) { + if p[i] < RuneSelf { + i++ + } else { + _, size := DecodeRune(p[i:]) + if size == 1 { + // All valid runes of size of 1 (those + // below RuneSelf) were handled above. + // This must be a RuneError. + return false + } + i += size + } + } + return true +} + +// ValidString reports whether s consists entirely of valid UTF-8-encoded runes. +func ValidString(s string) bool { + for i, r := range s { + if r == RuneError { + // The RuneError value can be an error + // sentinel value (if it's size 1) or the same + // value encoded properly. Decode it to see if + // it's the 1 byte sentinel value. + _, size := DecodeRuneInString(s[i:]) + if size == 1 { + return false + } + } + } + return true +} diff --git a/libgo/go/utf8/utf8_test.go b/libgo/go/utf8/utf8_test.go index 7a1db93..6cbbebc 100644 --- a/libgo/go/utf8/utf8_test.go +++ b/libgo/go/utf8/utf8_test.go @@ -274,6 +274,35 @@ func TestRuneCount(t *testing.T) { } } +type ValidTest struct { + in string + out bool +} + +var validTests = []ValidTest{ + {"", true}, + {"a", true}, + {"abc", true}, + {"Ж", true}, + {"ЖЖ", true}, + {"брэд-ЛГТМ", true}, + {"☺☻☹", true}, + {string([]byte{66, 250}), false}, + {string([]byte{66, 250, 67}), false}, + {"a\uFFFDb", true}, +} + +func TestValid(t *testing.T) { + for i, tt := range validTests { + if Valid([]byte(tt.in)) != tt.out { + t.Errorf("%d. Valid(%q) = %v; want %v", i, tt.in, !tt.out, tt.out) + } + if ValidString(tt.in) != tt.out { + t.Errorf("%d. ValidString(%q) = %v; want %v", i, tt.in, !tt.out, tt.out) + } + } +} + func BenchmarkRuneCountTenASCIIChars(b *testing.B) { for i := 0; i < b.N; i++ { RuneCountInString("0123456789") diff --git a/libgo/go/websocket/client.go b/libgo/go/websocket/client.go index 74bede4..b7eaafd 100644 --- a/libgo/go/websocket/client.go +++ b/libgo/go/websocket/client.go @@ -6,114 +6,119 @@ package websocket import ( "bufio" - "bytes" "crypto/tls" - "fmt" - "http" "io" "net" "os" - "rand" - "strings" "url" ) -type ProtocolError struct { - ErrorString string -} - -func (err *ProtocolError) String() string { return string(err.ErrorString) } - -var ( - ErrBadScheme = &ProtocolError{"bad scheme"} - ErrBadStatus = &ProtocolError{"bad status"} - ErrBadUpgrade = &ProtocolError{"missing or bad upgrade"} - ErrBadWebSocketOrigin = &ProtocolError{"missing or bad WebSocket-Origin"} - ErrBadWebSocketLocation = &ProtocolError{"missing or bad WebSocket-Location"} - ErrBadWebSocketProtocol = &ProtocolError{"missing or bad WebSocket-Protocol"} - ErrChallengeResponse = &ProtocolError{"mismatch challenge/response"} - secKeyRandomChars [0x30 - 0x21 + 0x7F - 0x3A]byte -) - +// DialError is an error that occurs while dialling a websocket server. type DialError struct { - URL string - Protocol string - Origin string - Error os.Error + *Config + Error os.Error } func (e *DialError) String() string { - return "websocket.Dial " + e.URL + ": " + e.Error.String() + return "websocket.Dial " + e.Config.Location.String() + ": " + e.Error.String() } -func init() { - i := 0 - for ch := byte(0x21); ch < 0x30; ch++ { - secKeyRandomChars[i] = ch - i++ +// NewConfig creates a new WebSocket config for client connection. +func NewConfig(server, origin string) (config *Config, err os.Error) { + config = new(Config) + config.Version = ProtocolVersionHybi13 + config.Location, err = url.ParseRequest(server) + if err != nil { + return } - for ch := byte(0x3a); ch < 0x7F; ch++ { - secKeyRandomChars[i] = ch - i++ + config.Origin, err = url.ParseRequest(origin) + if err != nil { + return } + return } -type handshaker func(resourceName, host, origin, location, protocol string, br *bufio.Reader, bw *bufio.Writer) os.Error - -// newClient creates a new Web Socket client connection. -func newClient(resourceName, host, origin, location, protocol string, rwc io.ReadWriteCloser, handshake handshaker) (ws *Conn, err os.Error) { +// NewClient creates a new WebSocket client connection over rwc. +func NewClient(config *Config, rwc io.ReadWriteCloser) (ws *Conn, err os.Error) { br := bufio.NewReader(rwc) bw := bufio.NewWriter(rwc) - err = handshake(resourceName, host, origin, location, protocol, br, bw) + switch config.Version { + case ProtocolVersionHixie75: + err = hixie75ClientHandshake(config, br, bw) + case ProtocolVersionHixie76, ProtocolVersionHybi00: + err = hixie76ClientHandshake(config, br, bw) + case ProtocolVersionHybi08, ProtocolVersionHybi13: + err = hybiClientHandshake(config, br, bw) + default: + err = ErrBadProtocolVersion + } if err != nil { return } buf := bufio.NewReadWriter(br, bw) - ws = newConn(origin, location, protocol, buf, rwc) + switch config.Version { + case ProtocolVersionHixie75, ProtocolVersionHixie76, ProtocolVersionHybi00: + ws = newHixieClientConn(config, buf, rwc) + case ProtocolVersionHybi08, ProtocolVersionHybi13: + ws = newHybiClientConn(config, buf, rwc) + } return } /* -Dial opens a new client connection to a Web Socket. +Dial opens a new client connection to a WebSocket. A trivial example client: package main import ( - "websocket" + "http" + "log" "strings" + "websocket" ) func main() { - ws, err := websocket.Dial("ws://localhost/ws", "", "http://localhost/"); - if err != nil { - panic("Dial: " + err.String()) + origin := "http://localhost/" + url := "ws://localhost/ws" + ws, err := websocket.Dial(url, "", origin) + if err != nil { + log.Fatal(err) } if _, err := ws.Write([]byte("hello, world!\n")); err != nil { - panic("Write: " + err.String()) + log.Fatal(err) } var msg = make([]byte, 512); if n, err := ws.Read(msg); err != nil { - panic("Read: " + err.String()) + log.Fatal(err) } // use msg[0:n] } */ func Dial(url_, protocol, origin string) (ws *Conn, err os.Error) { - var client net.Conn - - parsedUrl, err := url.Parse(url_) + config, err := NewConfig(url_, origin) if err != nil { - goto Error + return nil, err } + return DialConfig(config) +} - switch parsedUrl.Scheme { +// DialConfig opens a new client connection to a WebSocket with a config. +func DialConfig(config *Config) (ws *Conn, err os.Error) { + var client net.Conn + if config.Location == nil { + return nil, &DialError{config, ErrBadWebSocketLocation} + } + if config.Origin == nil { + return nil, &DialError{config, ErrBadWebSocketOrigin} + } + switch config.Location.Scheme { case "ws": - client, err = net.Dial("tcp", parsedUrl.Host) + client, err = net.Dial("tcp", config.Location.Host) case "wss": - client, err = tls.Dial("tcp", parsedUrl.Host, nil) + client, err = tls.Dial("tcp", config.Location.Host, config.TlsConfig) default: err = ErrBadScheme @@ -122,202 +127,12 @@ func Dial(url_, protocol, origin string) (ws *Conn, err os.Error) { goto Error } - ws, err = newClient(parsedUrl.RawPath, parsedUrl.Host, origin, url_, protocol, client, handshake) + ws, err = NewClient(config, client) if err != nil { goto Error } return Error: - return nil, &DialError{url_, protocol, origin, err} -} - -/* -Generates handshake key as described in 4.1 Opening handshake step 16 to 22. -cf. http://www.whatwg.org/specs/web-socket-protocol/ -*/ -func generateKeyNumber() (key string, number uint32) { - // 16. Let /spaces_n/ be a random integer from 1 to 12 inclusive. - spaces := rand.Intn(12) + 1 - - // 17. Let /max_n/ be the largest integer not greater than - // 4,294,967,295 divided by /spaces_n/ - max := int(4294967295 / uint32(spaces)) - - // 18. Let /number_n/ be a random integer from 0 to /max_n/ inclusive. - number = uint32(rand.Intn(max + 1)) - - // 19. Let /product_n/ be the result of multiplying /number_n/ and - // /spaces_n/ together. - product := number * uint32(spaces) - - // 20. Let /key_n/ be a string consisting of /product_n/, expressed - // in base ten using the numerals in the range U+0030 DIGIT ZERO (0) - // to U+0039 DIGIT NINE (9). - key = fmt.Sprintf("%d", product) - - // 21. Insert between one and twelve random characters from the ranges - // U+0021 to U+002F and U+003A to U+007E into /key_n/ at random - // positions. - n := rand.Intn(12) + 1 - for i := 0; i < n; i++ { - pos := rand.Intn(len(key)) + 1 - ch := secKeyRandomChars[rand.Intn(len(secKeyRandomChars))] - key = key[0:pos] + string(ch) + key[pos:] - } - - // 22. Insert /spaces_n/ U+0020 SPACE characters into /key_n/ at random - // positions other than the start or end of the string. - for i := 0; i < spaces; i++ { - pos := rand.Intn(len(key)-1) + 1 - key = key[0:pos] + " " + key[pos:] - } - - return -} - -/* -Generates handshake key_3 as described in 4.1 Opening handshake step 26. -cf. http://www.whatwg.org/specs/web-socket-protocol/ -*/ -func generateKey3() (key []byte) { - // 26. Let /key3/ be a string consisting of eight random bytes (or - // equivalently, a random 64 bit integer encoded in big-endian order). - key = make([]byte, 8) - for i := 0; i < 8; i++ { - key[i] = byte(rand.Intn(256)) - } - return -} - -/* -Web Socket protocol handshake based on -http://www.whatwg.org/specs/web-socket-protocol/ -(draft of http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol) -*/ -func handshake(resourceName, host, origin, location, protocol string, br *bufio.Reader, bw *bufio.Writer) (err os.Error) { - // 4.1. Opening handshake. - // Step 5. send a request line. - bw.WriteString("GET " + resourceName + " HTTP/1.1\r\n") - - // Step 6-14. push request headers in fields. - var fields []string - fields = append(fields, "Upgrade: WebSocket\r\n") - fields = append(fields, "Connection: Upgrade\r\n") - fields = append(fields, "Host: "+host+"\r\n") - fields = append(fields, "Origin: "+origin+"\r\n") - if protocol != "" { - fields = append(fields, "Sec-WebSocket-Protocol: "+protocol+"\r\n") - } - // TODO(ukai): Step 15. send cookie if any. - - // Step 16-23. generate keys and push Sec-WebSocket-Key<n> in fields. - key1, number1 := generateKeyNumber() - key2, number2 := generateKeyNumber() - fields = append(fields, "Sec-WebSocket-Key1: "+key1+"\r\n") - fields = append(fields, "Sec-WebSocket-Key2: "+key2+"\r\n") - - // Step 24. shuffle fields and send them out. - for i := 1; i < len(fields); i++ { - j := rand.Intn(i) - fields[i], fields[j] = fields[j], fields[i] - } - for i := 0; i < len(fields); i++ { - bw.WriteString(fields[i]) - } - // Step 25. send CRLF. - bw.WriteString("\r\n") - - // Step 26. generate 8 bytes random key. - key3 := generateKey3() - // Step 27. send it out. - bw.Write(key3) - if err = bw.Flush(); err != nil { - return - } - - // Step 28-29, 32-40. read response from server. - resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) - if err != nil { - return err - } - // Step 30. check response code is 101. - if resp.StatusCode != 101 { - return ErrBadStatus - } - - // Step 41. check websocket headers. - if resp.Header.Get("Upgrade") != "WebSocket" || - strings.ToLower(resp.Header.Get("Connection")) != "upgrade" { - return ErrBadUpgrade - } - - if resp.Header.Get("Sec-Websocket-Origin") != origin { - return ErrBadWebSocketOrigin - } - - if resp.Header.Get("Sec-Websocket-Location") != location { - return ErrBadWebSocketLocation - } - - if protocol != "" && resp.Header.Get("Sec-Websocket-Protocol") != protocol { - return ErrBadWebSocketProtocol - } - - // Step 42-43. get expected data from challenge data. - expected, err := getChallengeResponse(number1, number2, key3) - if err != nil { - return err - } - - // Step 44. read 16 bytes from server. - reply := make([]byte, 16) - if _, err = io.ReadFull(br, reply); err != nil { - return err - } - - // Step 45. check the reply equals to expected data. - if !bytes.Equal(expected, reply) { - return ErrChallengeResponse - } - // WebSocket connection is established. - return -} - -/* -Handshake described in (soon obsolete) -draft-hixie-thewebsocket-protocol-75. -*/ -func draft75handshake(resourceName, host, origin, location, protocol string, br *bufio.Reader, bw *bufio.Writer) (err os.Error) { - bw.WriteString("GET " + resourceName + " HTTP/1.1\r\n") - bw.WriteString("Upgrade: WebSocket\r\n") - bw.WriteString("Connection: Upgrade\r\n") - bw.WriteString("Host: " + host + "\r\n") - bw.WriteString("Origin: " + origin + "\r\n") - if protocol != "" { - bw.WriteString("WebSocket-Protocol: " + protocol + "\r\n") - } - bw.WriteString("\r\n") - bw.Flush() - resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) - if err != nil { - return - } - if resp.Status != "101 Web Socket Protocol Handshake" { - return ErrBadStatus - } - if resp.Header.Get("Upgrade") != "WebSocket" || - resp.Header.Get("Connection") != "Upgrade" { - return ErrBadUpgrade - } - if resp.Header.Get("Websocket-Origin") != origin { - return ErrBadWebSocketOrigin - } - if resp.Header.Get("Websocket-Location") != location { - return ErrBadWebSocketLocation - } - if protocol != "" && resp.Header.Get("Websocket-Protocol") != protocol { - return ErrBadWebSocketProtocol - } - return + return nil, &DialError{config, err} } diff --git a/libgo/go/websocket/hixie.go b/libgo/go/websocket/hixie.go new file mode 100644 index 0000000..841ff3c --- /dev/null +++ b/libgo/go/websocket/hixie.go @@ -0,0 +1,696 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +// This file implements a protocol of Hixie draft version 75 and 76 +// (draft 76 equals to hybi 00) + +import ( + "bufio" + "bytes" + "crypto/md5" + "encoding/binary" + "fmt" + "http" + "io" + "io/ioutil" + "os" + "rand" + "strconv" + "strings" + "url" +) + +// An aray of characters to be randomly inserted to construct Sec-WebSocket-Key +// value. It holds characters from ranges U+0021 to U+002F and U+003A to U+007E. +// See Step 21 in Section 4.1 Opening handshake. +// http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00#page-22 +var secKeyRandomChars [0x30 - 0x21 + 0x7F - 0x3A]byte + +func init() { + i := 0 + for ch := byte(0x21); ch < 0x30; ch++ { + secKeyRandomChars[i] = ch + i++ + } + for ch := byte(0x3a); ch < 0x7F; ch++ { + secKeyRandomChars[i] = ch + i++ + } +} + +type byteReader interface { + ReadByte() (byte, os.Error) +} + +// readHixieLength reads frame length for frame type 0x80-0xFF +// as defined in Hixie draft. +// See section 4.2 Data framing. +// http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00#section-4.2 +func readHixieLength(r byteReader) (length int64, lengthFields []byte, err os.Error) { + for { + c, err := r.ReadByte() + if err != nil { + return 0, nil, err + } + lengthFields = append(lengthFields, c) + length = length*128 + int64(c&0x7f) + if c&0x80 == 0 { + break + } + } + return +} + +// A hixieLengthFrameReader is a reader for frame type 0x80-0xFF +// as defined in hixie draft. +type hixieLengthFrameReader struct { + reader io.Reader + FrameType byte + Length int64 + header *bytes.Buffer + length int +} + +func (frame *hixieLengthFrameReader) Read(msg []byte) (n int, err os.Error) { + return frame.reader.Read(msg) +} + +func (frame *hixieLengthFrameReader) PayloadType() byte { + if frame.FrameType == '\xff' && frame.Length == 0 { + return CloseFrame + } + return UnknownFrame +} + +func (frame *hixieLengthFrameReader) HeaderReader() io.Reader { + if frame.header == nil { + return nil + } + if frame.header.Len() == 0 { + frame.header = nil + return nil + } + return frame.header +} + +func (frame *hixieLengthFrameReader) TrailerReader() io.Reader { return nil } + +func (frame *hixieLengthFrameReader) Len() (n int) { return frame.length } + +// A HixieSentinelFrameReader is a reader for frame type 0x00-0x7F +// as defined in hixie draft. +type hixieSentinelFrameReader struct { + reader *bufio.Reader + FrameType byte + header *bytes.Buffer + data []byte + seenTrailer bool + trailer *bytes.Buffer +} + +func (frame *hixieSentinelFrameReader) Read(msg []byte) (n int, err os.Error) { + if len(frame.data) == 0 { + if frame.seenTrailer { + return 0, os.EOF + } + frame.data, err = frame.reader.ReadSlice('\xff') + if err == nil { + frame.seenTrailer = true + frame.data = frame.data[:len(frame.data)-1] // trim \xff + frame.trailer = bytes.NewBuffer([]byte{0xff}) + } + } + n = copy(msg, frame.data) + frame.data = frame.data[n:] + return n, err +} + +func (frame *hixieSentinelFrameReader) PayloadType() byte { + if frame.FrameType == 0 { + return TextFrame + } + return UnknownFrame +} + +func (frame *hixieSentinelFrameReader) HeaderReader() io.Reader { + if frame.header == nil { + return nil + } + if frame.header.Len() == 0 { + frame.header = nil + return nil + } + return frame.header +} + +func (frame *hixieSentinelFrameReader) TrailerReader() io.Reader { + if frame.trailer == nil { + return nil + } + if frame.trailer.Len() == 0 { + frame.trailer = nil + return nil + } + return frame.trailer +} + +func (frame *hixieSentinelFrameReader) Len() int { return -1 } + +// A HixieFrameReaderFactory creates new frame reader based on its frame type. +type hixieFrameReaderFactory struct { + *bufio.Reader +} + +func (buf hixieFrameReaderFactory) NewFrameReader() (r frameReader, err os.Error) { + var header []byte + var b byte + b, err = buf.ReadByte() + if err != nil { + return + } + header = append(header, b) + if b&0x80 == 0x80 { + length, lengthFields, err := readHixieLength(buf.Reader) + if err != nil { + return nil, err + } + if length == 0 { + return nil, os.EOF + } + header = append(header, lengthFields...) + return &hixieLengthFrameReader{ + reader: io.LimitReader(buf.Reader, length), + FrameType: b, + Length: length, + header: bytes.NewBuffer(header)}, err + } + return &hixieSentinelFrameReader{ + reader: buf.Reader, + FrameType: b, + header: bytes.NewBuffer(header)}, err +} + +type hixiFrameWriter struct { + writer *bufio.Writer +} + +func (frame *hixiFrameWriter) Write(msg []byte) (n int, err os.Error) { + frame.writer.WriteByte(0) + frame.writer.Write(msg) + frame.writer.WriteByte(0xff) + err = frame.writer.Flush() + return len(msg), err +} + +func (frame *hixiFrameWriter) Close() os.Error { return nil } + +type hixiFrameWriterFactory struct { + *bufio.Writer +} + +func (buf hixiFrameWriterFactory) NewFrameWriter(payloadType byte) (frame frameWriter, err os.Error) { + if payloadType != TextFrame { + return nil, ErrNotSupported + } + return &hixiFrameWriter{writer: buf.Writer}, nil +} + +type hixiFrameHandler struct { + conn *Conn +} + +func (handler *hixiFrameHandler) HandleFrame(frame frameReader) (r frameReader, err os.Error) { + if header := frame.HeaderReader(); header != nil { + io.Copy(ioutil.Discard, header) + } + if frame.PayloadType() != TextFrame { + io.Copy(ioutil.Discard, frame) + return nil, nil + } + return frame, nil +} + +func (handler *hixiFrameHandler) WriteClose(_ int) (err os.Error) { + handler.conn.wio.Lock() + defer handler.conn.wio.Unlock() + closingFrame := []byte{'\xff', '\x00'} + handler.conn.buf.Write(closingFrame) + return handler.conn.buf.Flush() +} + +// newHixiConn creates a new WebSocket connection speaking hixie draft protocol. +func newHixieConn(config *Config, buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) *Conn { + if buf == nil { + br := bufio.NewReader(rwc) + bw := bufio.NewWriter(rwc) + buf = bufio.NewReadWriter(br, bw) + } + ws := &Conn{config: config, request: request, buf: buf, rwc: rwc, + frameReaderFactory: hixieFrameReaderFactory{buf.Reader}, + frameWriterFactory: hixiFrameWriterFactory{buf.Writer}, + PayloadType: TextFrame} + ws.frameHandler = &hixiFrameHandler{ws} + return ws +} + +// getChallengeResponse computes the expected response from the +// challenge as described in section 5.1 Opening Handshake steps 42 to +// 43 of http://www.whatwg.org/specs/web-socket-protocol/ +func getChallengeResponse(number1, number2 uint32, key3 []byte) (expected []byte, err os.Error) { + // 41. Let /challenge/ be the concatenation of /number_1/, expressed + // a big-endian 32 bit integer, /number_2/, expressed in a big- + // endian 32 bit integer, and the eight bytes of /key_3/ in the + // order they were sent to the wire. + challenge := make([]byte, 16) + binary.BigEndian.PutUint32(challenge[0:], number1) + binary.BigEndian.PutUint32(challenge[4:], number2) + copy(challenge[8:], key3) + + // 42. Let /expected/ be the MD5 fingerprint of /challenge/ as a big- + // endian 128 bit string. + h := md5.New() + if _, err = h.Write(challenge); err != nil { + return + } + expected = h.Sum() + return +} + +// Generates handshake key as described in 4.1 Opening handshake step 16 to 22. +// cf. http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +func generateKeyNumber() (key string, number uint32) { + // 16. Let /spaces_n/ be a random integer from 1 to 12 inclusive. + spaces := rand.Intn(12) + 1 + + // 17. Let /max_n/ be the largest integer not greater than + // 4,294,967,295 divided by /spaces_n/ + max := int(4294967295 / uint32(spaces)) + + // 18. Let /number_n/ be a random integer from 0 to /max_n/ inclusive. + number = uint32(rand.Intn(max + 1)) + + // 19. Let /product_n/ be the result of multiplying /number_n/ and + // /spaces_n/ together. + product := number * uint32(spaces) + + // 20. Let /key_n/ be a string consisting of /product_n/, expressed + // in base ten using the numerals in the range U+0030 DIGIT ZERO (0) + // to U+0039 DIGIT NINE (9). + key = fmt.Sprintf("%d", product) + + // 21. Insert between one and twelve random characters from the ranges + // U+0021 to U+002F and U+003A to U+007E into /key_n/ at random + // positions. + n := rand.Intn(12) + 1 + for i := 0; i < n; i++ { + pos := rand.Intn(len(key)) + 1 + ch := secKeyRandomChars[rand.Intn(len(secKeyRandomChars))] + key = key[0:pos] + string(ch) + key[pos:] + } + + // 22. Insert /spaces_n/ U+0020 SPACE characters into /key_n/ at random + // positions other than the start or end of the string. + for i := 0; i < spaces; i++ { + pos := rand.Intn(len(key)-1) + 1 + key = key[0:pos] + " " + key[pos:] + } + + return +} + +// Generates handshake key_3 as described in 4.1 Opening handshake step 26. +// cf. http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +func generateKey3() (key []byte) { + // 26. Let /key3/ be a string consisting of eight random bytes (or + // equivalently, a random 64 bit integer encoded in big-endian order). + key = make([]byte, 8) + for i := 0; i < 8; i++ { + key[i] = byte(rand.Intn(256)) + } + return +} + +// Cilent handhake described in (soon obsolete) +// draft-ietf-hybi-thewebsocket-protocol-00 +// (draft-hixie-thewebsocket-protocol-76) +func hixie76ClientHandshake(config *Config, br *bufio.Reader, bw *bufio.Writer) (err os.Error) { + switch config.Version { + case ProtocolVersionHixie76, ProtocolVersionHybi00: + default: + panic("wrong protocol version.") + } + // 4.1. Opening handshake. + // Step 5. send a request line. + bw.WriteString("GET " + config.Location.RawPath + " HTTP/1.1\r\n") + + // Step 6-14. push request headers in fields. + fields := []string{ + "Upgrade: WebSocket\r\n", + "Connection: Upgrade\r\n", + "Host: " + config.Location.Host + "\r\n", + "Origin: " + config.Origin.String() + "\r\n", + } + if len(config.Protocol) > 0 { + if len(config.Protocol) != 1 { + return ErrBadWebSocketProtocol + } + fields = append(fields, "Sec-WebSocket-Protocol: "+config.Protocol[0]+"\r\n") + } + // TODO(ukai): Step 15. send cookie if any. + + // Step 16-23. generate keys and push Sec-WebSocket-Key<n> in fields. + key1, number1 := generateKeyNumber() + key2, number2 := generateKeyNumber() + if config.handshakeData != nil { + key1 = config.handshakeData["key1"] + n, err := strconv.Atoui(config.handshakeData["number1"]) + if err != nil { + panic(err) + } + number1 = uint32(n) + key2 = config.handshakeData["key2"] + n, err = strconv.Atoui(config.handshakeData["number2"]) + if err != nil { + panic(err) + } + number2 = uint32(n) + } + fields = append(fields, "Sec-WebSocket-Key1: "+key1+"\r\n") + fields = append(fields, "Sec-WebSocket-Key2: "+key2+"\r\n") + + // Step 24. shuffle fields and send them out. + for i := 1; i < len(fields); i++ { + j := rand.Intn(i) + fields[i], fields[j] = fields[j], fields[i] + } + for i := 0; i < len(fields); i++ { + bw.WriteString(fields[i]) + } + // Step 25. send CRLF. + bw.WriteString("\r\n") + + // Step 26. generate 8 bytes random key. + key3 := generateKey3() + if config.handshakeData != nil { + key3 = []byte(config.handshakeData["key3"]) + } + // Step 27. send it out. + bw.Write(key3) + if err = bw.Flush(); err != nil { + return + } + + // Step 28-29, 32-40. read response from server. + resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) + if err != nil { + return err + } + // Step 30. check response code is 101. + if resp.StatusCode != 101 { + return ErrBadStatus + } + + // Step 41. check websocket headers. + if resp.Header.Get("Upgrade") != "WebSocket" || + strings.ToLower(resp.Header.Get("Connection")) != "upgrade" { + return ErrBadUpgrade + } + + if resp.Header.Get("Sec-Websocket-Origin") != config.Origin.String() { + return ErrBadWebSocketOrigin + } + + if resp.Header.Get("Sec-Websocket-Location") != config.Location.String() { + return ErrBadWebSocketLocation + } + + if len(config.Protocol) > 0 && resp.Header.Get("Sec-Websocket-Protocol") != config.Protocol[0] { + return ErrBadWebSocketProtocol + } + + // Step 42-43. get expected data from challenge data. + expected, err := getChallengeResponse(number1, number2, key3) + if err != nil { + return err + } + + // Step 44. read 16 bytes from server. + reply := make([]byte, 16) + if _, err = io.ReadFull(br, reply); err != nil { + return err + } + + // Step 45. check the reply equals to expected data. + if !bytes.Equal(expected, reply) { + return ErrChallengeResponse + } + // WebSocket connection is established. + return +} + +// Client Handshake described in (soon obsolete) +// draft-hixie-thewebsocket-protocol-75. +func hixie75ClientHandshake(config *Config, br *bufio.Reader, bw *bufio.Writer) (err os.Error) { + if config.Version != ProtocolVersionHixie75 { + panic("wrong protocol version.") + } + bw.WriteString("GET " + config.Location.RawPath + " HTTP/1.1\r\n") + bw.WriteString("Upgrade: WebSocket\r\n") + bw.WriteString("Connection: Upgrade\r\n") + bw.WriteString("Host: " + config.Location.Host + "\r\n") + bw.WriteString("Origin: " + config.Origin.String() + "\r\n") + if len(config.Protocol) > 0 { + if len(config.Protocol) != 1 { + return ErrBadWebSocketProtocol + } + bw.WriteString("WebSocket-Protocol: " + config.Protocol[0] + "\r\n") + } + bw.WriteString("\r\n") + bw.Flush() + resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) + if err != nil { + return + } + if resp.Status != "101 Web Socket Protocol Handshake" { + return ErrBadStatus + } + if resp.Header.Get("Upgrade") != "WebSocket" || + resp.Header.Get("Connection") != "Upgrade" { + return ErrBadUpgrade + } + if resp.Header.Get("Websocket-Origin") != config.Origin.String() { + return ErrBadWebSocketOrigin + } + if resp.Header.Get("Websocket-Location") != config.Location.String() { + return ErrBadWebSocketLocation + } + if len(config.Protocol) > 0 && resp.Header.Get("Websocket-Protocol") != config.Protocol[0] { + return ErrBadWebSocketProtocol + } + return +} + +// newHixieClientConn returns new WebSocket connection speaking hixie draft protocol. +func newHixieClientConn(config *Config, buf *bufio.ReadWriter, rwc io.ReadWriteCloser) *Conn { + return newHixieConn(config, buf, rwc, nil) +} + +// Gets key number from Sec-WebSocket-Key<n>: field as described +// in 5.2 Sending the server's opening handshake, 4. +func getKeyNumber(s string) (r uint32) { + // 4. Let /key-number_n/ be the digits (characters in the range + // U+0030 DIGIT ZERO (0) to U+0039 DIGIT NINE (9)) in /key_1/, + // interpreted as a base ten integer, ignoring all other characters + // in /key_n/. + r = 0 + for i := 0; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + r = r*10 + uint32(s[i]) - '0' + } + } + return +} + +// A Hixie76ServerHandshaker performs a server handshake using +// hixie draft 76 protocol. +type hixie76ServerHandshaker struct { + *Config + challengeResponse []byte +} + +func (c *hixie76ServerHandshaker) ReadHandshake(buf *bufio.Reader, req *http.Request) (code int, err os.Error) { + c.Version = ProtocolVersionHybi00 + if req.Method != "GET" { + return http.StatusMethodNotAllowed, ErrBadRequestMethod + } + // HTTP version can be safely ignored. + + if strings.ToLower(req.Header.Get("Upgrade")) != "websocket" || + strings.ToLower(req.Header.Get("Connection")) != "upgrade" { + return http.StatusBadRequest, ErrNotWebSocket + } + + // TODO(ukai): check Host + c.Origin, err = url.ParseRequest(req.Header.Get("Origin")) + if err != nil { + return http.StatusBadRequest, err + } + + key1 := req.Header.Get("Sec-Websocket-Key1") + if key1 == "" { + return http.StatusBadRequest, ErrChallengeResponse + } + key2 := req.Header.Get("Sec-Websocket-Key2") + if key2 == "" { + return http.StatusBadRequest, ErrChallengeResponse + } + key3 := make([]byte, 8) + if _, err := io.ReadFull(buf, key3); err != nil { + return http.StatusBadRequest, ErrChallengeResponse + } + + var scheme string + if req.TLS != nil { + scheme = "wss" + } else { + scheme = "ws" + } + c.Location, err = url.ParseRequest(scheme + "://" + req.Host + req.URL.RawPath) + if err != nil { + return http.StatusBadRequest, err + } + + // Step 4. get key number in Sec-WebSocket-Key<n> fields. + keyNumber1 := getKeyNumber(key1) + keyNumber2 := getKeyNumber(key2) + + // Step 5. get number of spaces in Sec-WebSocket-Key<n> fields. + space1 := uint32(strings.Count(key1, " ")) + space2 := uint32(strings.Count(key2, " ")) + if space1 == 0 || space2 == 0 { + return http.StatusBadRequest, ErrChallengeResponse + } + + // Step 6. key number must be an integral multiple of spaces. + if keyNumber1%space1 != 0 || keyNumber2%space2 != 0 { + return http.StatusBadRequest, ErrChallengeResponse + } + + // Step 7. let part be key number divided by spaces. + part1 := keyNumber1 / space1 + part2 := keyNumber2 / space2 + + // Step 8. let challenge be concatenation of part1, part2 and key3. + // Step 9. get MD5 fingerprint of challenge. + c.challengeResponse, err = getChallengeResponse(part1, part2, key3) + if err != nil { + return http.StatusInternalServerError, err + } + protocol := strings.TrimSpace(req.Header.Get("Sec-Websocket-Protocol")) + protocols := strings.Split(protocol, ",") + for i := 0; i < len(protocols); i++ { + c.Protocol = append(c.Protocol, strings.TrimSpace(protocols[i])) + } + + return http.StatusSwitchingProtocols, nil +} + +func (c *hixie76ServerHandshaker) AcceptHandshake(buf *bufio.Writer) (err os.Error) { + if len(c.Protocol) > 0 { + if len(c.Protocol) != 1 { + return ErrBadWebSocketProtocol + } + } + + // Step 10. send response status line. + buf.WriteString("HTTP/1.1 101 WebSocket Protocol Handshake\r\n") + // Step 11. send response headers. + buf.WriteString("Upgrade: WebSocket\r\n") + buf.WriteString("Connection: Upgrade\r\n") + buf.WriteString("Sec-WebSocket-Origin: " + c.Origin.String() + "\r\n") + buf.WriteString("Sec-WebSocket-Location: " + c.Location.String() + "\r\n") + if len(c.Protocol) > 0 { + buf.WriteString("Sec-WebSocket-Protocol: " + c.Protocol[0] + "\r\n") + } + // Step 12. send CRLF. + buf.WriteString("\r\n") + // Step 13. send response data. + buf.Write(c.challengeResponse) + return buf.Flush() +} + +func (c *hixie76ServerHandshaker) NewServerConn(buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) (conn *Conn) { + return newHixieServerConn(c.Config, buf, rwc, request) +} + +// A hixie75ServerHandshaker performs a server handshake using +// hixie draft 75 protocol. +type hixie75ServerHandshaker struct { + *Config +} + +func (c *hixie75ServerHandshaker) ReadHandshake(buf *bufio.Reader, req *http.Request) (code int, err os.Error) { + c.Version = ProtocolVersionHixie75 + if req.Method != "GET" || req.Proto != "HTTP/1.1" { + return http.StatusMethodNotAllowed, ErrBadRequestMethod + } + if req.Header.Get("Upgrade") != "WebSocket" { + return http.StatusBadRequest, ErrNotWebSocket + } + if req.Header.Get("Connection") != "Upgrade" { + return http.StatusBadRequest, ErrNotWebSocket + } + c.Origin, err = url.ParseRequest(strings.TrimSpace(req.Header.Get("Origin"))) + if err != nil { + return http.StatusBadRequest, err + } + + var scheme string + if req.TLS != nil { + scheme = "wss" + } else { + scheme = "ws" + } + c.Location, err = url.ParseRequest(scheme + "://" + req.Host + req.URL.RawPath) + if err != nil { + return http.StatusBadRequest, err + } + protocol := strings.TrimSpace(req.Header.Get("Websocket-Protocol")) + protocols := strings.Split(protocol, ",") + for i := 0; i < len(protocols); i++ { + c.Protocol = append(c.Protocol, strings.TrimSpace(protocols[i])) + } + + return http.StatusSwitchingProtocols, nil +} + +func (c *hixie75ServerHandshaker) AcceptHandshake(buf *bufio.Writer) (err os.Error) { + if len(c.Protocol) > 0 { + if len(c.Protocol) != 1 { + return ErrBadWebSocketProtocol + } + } + + buf.WriteString("HTTP/1.1 101 Web Socket Protocol Handshake\r\n") + buf.WriteString("Upgrade: WebSocket\r\n") + buf.WriteString("Connection: Upgrade\r\n") + buf.WriteString("WebSocket-Origin: " + c.Origin.String() + "\r\n") + buf.WriteString("WebSocket-Location: " + c.Location.String() + "\r\n") + if len(c.Protocol) > 0 { + buf.WriteString("WebSocket-Protocol: " + c.Protocol[0] + "\r\n") + } + buf.WriteString("\r\n") + return buf.Flush() +} + +func (c *hixie75ServerHandshaker) NewServerConn(buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) (conn *Conn) { + return newHixieServerConn(c.Config, buf, rwc, request) +} + +// newHixieServerConn returns a new WebSocket connection speaking hixie draft protocol. +func newHixieServerConn(config *Config, buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) *Conn { + return newHixieConn(config, buf, rwc, request) +} diff --git a/libgo/go/websocket/hixie_test.go b/libgo/go/websocket/hixie_test.go new file mode 100644 index 0000000..98a0de4 --- /dev/null +++ b/libgo/go/websocket/hixie_test.go @@ -0,0 +1,201 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "bytes" + "fmt" + "http" + "os" + "strings" + "testing" + "url" +) + +// Test the getChallengeResponse function with values from section +// 5.1 of the specification steps 18, 26, and 43 from +// http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +func TestHixie76Challenge(t *testing.T) { + var part1 uint32 = 777007543 + var part2 uint32 = 114997259 + key3 := []byte{0x47, 0x30, 0x22, 0x2D, 0x5A, 0x3F, 0x47, 0x58} + expected := []byte("0st3Rl&q-2ZU^weu") + + response, err := getChallengeResponse(part1, part2, key3) + if err != nil { + t.Errorf("getChallengeResponse: returned error %v", err) + return + } + if !bytes.Equal(expected, response) { + t.Errorf("getChallengeResponse: expected %q got %q", expected, response) + } +} + +func TestHixie76ClientHandshake(t *testing.T) { + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + br := bufio.NewReader(strings.NewReader(`HTTP/1.1 101 WebSocket Protocol Handshake +Upgrade: WebSocket +Connection: Upgrade +Sec-WebSocket-Origin: http://example.com +Sec-WebSocket-Location: ws://example.com/demo +Sec-WebSocket-Protocol: sample + +8jKS'y:G*Co,Wxa-`)) + + var err os.Error + config := new(Config) + config.Location, err = url.ParseRequest("ws://example.com/demo") + if err != nil { + t.Fatal("location url", err) + } + config.Origin, err = url.ParseRequest("http://example.com") + if err != nil { + t.Fatal("origin url", err) + } + config.Protocol = append(config.Protocol, "sample") + config.Version = ProtocolVersionHixie76 + + config.handshakeData = map[string]string{ + "key1": "4 @1 46546xW%0l 1 5", + "number1": "829309203", + "key2": "12998 5 Y3 1 .P00", + "number2": "259970620", + "key3": "^n:ds[4U", + } + err = hixie76ClientHandshake(config, br, bw) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + req, err := http.ReadRequest(bufio.NewReader(b)) + if err != nil { + t.Fatalf("read request: %v", err) + } + if req.Method != "GET" { + t.Errorf("request method expected GET, but got %q", req.Method) + } + if req.URL.Path != "/demo" { + t.Errorf("request path expected /demo, but got %q", req.URL.Path) + } + if req.Proto != "HTTP/1.1" { + t.Errorf("request proto expected HTTP/1.1, but got %q", req.Proto) + } + if req.Host != "example.com" { + t.Errorf("request Host expected example.com, but got %v", req.Host) + } + var expectedHeader = map[string]string{ + "Connection": "Upgrade", + "Upgrade": "WebSocket", + "Origin": "http://example.com", + "Sec-Websocket-Key1": config.handshakeData["key1"], + "Sec-Websocket-Key2": config.handshakeData["key2"], + "Sec-WebSocket-Protocol": config.Protocol[0], + } + for k, v := range expectedHeader { + if req.Header.Get(k) != v { + t.Errorf(fmt.Sprintf("%s expected %q but got %q", k, v, req.Header.Get(k))) + } + } +} + +func TestHixie76ServerHandshake(t *testing.T) { + config := new(Config) + handshaker := &hixie76ServerHandshaker{Config: config} + br := bufio.NewReader(strings.NewReader(`GET /demo HTTP/1.1 +Host: example.com +Connection: Upgrade +Sec-WebSocket-Key2: 12998 5 Y3 1 .P00 +Sec-WebSocket-Protocol: sample +Upgrade: WebSocket +Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5 +Origin: http://example.com + +^n:ds[4U`)) + req, err := http.ReadRequest(br) + if err != nil { + t.Fatal("request", err) + } + code, err := handshaker.ReadHandshake(br, req) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + if code != http.StatusSwitchingProtocols { + t.Errorf("status expected %q but got %q", http.StatusSwitchingProtocols, code) + } + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + + err = handshaker.AcceptHandshake(bw) + if err != nil { + t.Errorf("handshake response failed: %v", err) + } + expectedResponse := strings.Join([]string{ + "HTTP/1.1 101 WebSocket Protocol Handshake", + "Upgrade: WebSocket", + "Connection: Upgrade", + "Sec-WebSocket-Origin: http://example.com", + "Sec-WebSocket-Location: ws://example.com/demo", + "Sec-WebSocket-Protocol: sample", + "", ""}, "\r\n") + "8jKS'y:G*Co,Wxa-" + if b.String() != expectedResponse { + t.Errorf("handshake expected %q but got %q", expectedResponse, b.String()) + } +} + +func TestHixie76SkipLengthFrame(t *testing.T) { + b := []byte{'\x80', '\x01', 'x', 0, 'h', 'e', 'l', 'l', 'o', '\xff'} + buf := bytes.NewBuffer(b) + br := bufio.NewReader(buf) + bw := bufio.NewWriter(buf) + config := newConfig(t, "/") + ws := newHixieConn(config, bufio.NewReadWriter(br, bw), nil, nil) + msg := make([]byte, 5) + n, err := ws.Read(msg) + if err != nil { + t.Errorf("Read: %v", err) + } + if !bytes.Equal(b[4:9], msg[0:n]) { + t.Errorf("Read: expected %q got %q", b[4:9], msg[0:n]) + } +} + +func TestHixie76SkipNoUTF8Frame(t *testing.T) { + b := []byte{'\x01', 'n', '\xff', 0, 'h', 'e', 'l', 'l', 'o', '\xff'} + buf := bytes.NewBuffer(b) + br := bufio.NewReader(buf) + bw := bufio.NewWriter(buf) + config := newConfig(t, "/") + ws := newHixieConn(config, bufio.NewReadWriter(br, bw), nil, nil) + msg := make([]byte, 5) + n, err := ws.Read(msg) + if err != nil { + t.Errorf("Read: %v", err) + } + if !bytes.Equal(b[4:9], msg[0:n]) { + t.Errorf("Read: expected %q got %q", b[4:9], msg[0:n]) + } +} + +func TestHixie76ClosingFrame(t *testing.T) { + b := []byte{0, 'h', 'e', 'l', 'l', 'o', '\xff'} + buf := bytes.NewBuffer(b) + br := bufio.NewReader(buf) + bw := bufio.NewWriter(buf) + config := newConfig(t, "/") + ws := newHixieConn(config, bufio.NewReadWriter(br, bw), nil, nil) + msg := make([]byte, 5) + n, err := ws.Read(msg) + if err != nil { + t.Errorf("read: %v", err) + } + if !bytes.Equal(b[1:6], msg[0:n]) { + t.Errorf("Read: expected %q got %q", b[1:6], msg[0:n]) + } + n, err = ws.Read(msg) + if err != os.EOF { + t.Errorf("read: %v", err) + } +} diff --git a/libgo/go/websocket/hybi.go b/libgo/go/websocket/hybi.go new file mode 100644 index 0000000..fe08b3d --- /dev/null +++ b/libgo/go/websocket/hybi.go @@ -0,0 +1,550 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +// This file implements a protocol of hybi draft. +// http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-17 + +import ( + "bufio" + "bytes" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "encoding/binary" + "fmt" + "http" + "io" + "io/ioutil" + "os" + "strings" + "url" +) + +const ( + websocketGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + closeStatusNormal = 1000 + closeStatusGoingAway = 1001 + closeStatusProtocolError = 1002 + closeStatusUnsupportedData = 1003 + closeStatusFrameTooLarge = 1004 + closeStatusNoStatusRcvd = 1005 + closeStatusAbnormalClosure = 1006 + closeStatusBadMessageData = 1007 + closeStatusPolicyViolation = 1008 + closeStatusTooBigData = 1009 + closeStatusExtensionMismatch = 1010 + + maxControlFramePayloadLength = 125 +) + +var ( + ErrBadMaskingKey = &ProtocolError{"bad masking key"} + ErrBadPongMessage = &ProtocolError{"bad pong message"} + ErrBadClosingStatus = &ProtocolError{"bad closing status"} + ErrUnsupportedExtensions = &ProtocolError{"unsupported extensions"} + ErrNotImplemented = &ProtocolError{"not implemented"} +) + +// A hybiFrameHeader is a frame header as defined in hybi draft. +type hybiFrameHeader struct { + Fin bool + Rsv [3]bool + OpCode byte + Length int64 + MaskingKey []byte + + data *bytes.Buffer +} + +// A hybiFrameReader is a reader for hybi frame. +type hybiFrameReader struct { + reader io.Reader + + header hybiFrameHeader + pos int64 + length int +} + +func (frame *hybiFrameReader) Read(msg []byte) (n int, err os.Error) { + n, err = frame.reader.Read(msg) + if err != nil { + return 0, err + } + if frame.header.MaskingKey != nil { + for i := 0; i < n; i++ { + msg[i] = msg[i] ^ frame.header.MaskingKey[frame.pos%4] + frame.pos++ + } + } + return n, err +} + +func (frame *hybiFrameReader) PayloadType() byte { return frame.header.OpCode } + +func (frame *hybiFrameReader) HeaderReader() io.Reader { + if frame.header.data == nil { + return nil + } + if frame.header.data.Len() == 0 { + return nil + } + return frame.header.data +} + +func (frame *hybiFrameReader) TrailerReader() io.Reader { return nil } + +func (frame *hybiFrameReader) Len() (n int) { return frame.length } + +// A hybiFrameReaderFactory creates new frame reader based on its frame type. +type hybiFrameReaderFactory struct { + *bufio.Reader +} + +// NewFrameReader reads a frame header from the connection, and creates new reader for the frame. +// See Section 5.2 Base Frameing protocol for detail. +// http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-17#section-5.2 +func (buf hybiFrameReaderFactory) NewFrameReader() (frame frameReader, err os.Error) { + hybiFrame := new(hybiFrameReader) + frame = hybiFrame + var header []byte + var b byte + // First byte. FIN/RSV1/RSV2/RSV3/OpCode(4bits) + b, err = buf.ReadByte() + if err != nil { + return + } + header = append(header, b) + hybiFrame.header.Fin = ((header[0] >> 7) & 1) != 0 + for i := 0; i < 3; i++ { + j := uint(6 - i) + hybiFrame.header.Rsv[i] = ((header[0] >> j) & 1) != 0 + } + hybiFrame.header.OpCode = header[0] & 0x0f + + // Second byte. Mask/Payload len(7bits) + b, err = buf.ReadByte() + if err != nil { + return + } + header = append(header, b) + mask := (b & 0x80) != 0 + b &= 0x7f + lengthFields := 0 + switch { + case b <= 125: // Payload length 7bits. + hybiFrame.header.Length = int64(b) + case b == 126: // Payload length 7+16bits + lengthFields = 2 + case b == 127: // Payload length 7+64bits + lengthFields = 8 + } + for i := 0; i < lengthFields; i++ { + b, err = buf.ReadByte() + if err != nil { + return + } + header = append(header, b) + hybiFrame.header.Length = hybiFrame.header.Length*256 + int64(b) + } + if mask { + // Masking key. 4 bytes. + for i := 0; i < 4; i++ { + b, err = buf.ReadByte() + if err != nil { + return + } + header = append(header, b) + hybiFrame.header.MaskingKey = append(hybiFrame.header.MaskingKey, b) + } + } + hybiFrame.reader = io.LimitReader(buf.Reader, hybiFrame.header.Length) + hybiFrame.header.data = bytes.NewBuffer(header) + hybiFrame.length = len(header) + int(hybiFrame.header.Length) + return +} + +// A HybiFrameWriter is a writer for hybi frame. +type hybiFrameWriter struct { + writer *bufio.Writer + + header *hybiFrameHeader +} + +func (frame *hybiFrameWriter) Write(msg []byte) (n int, err os.Error) { + var header []byte + var b byte + if frame.header.Fin { + b |= 0x80 + } + for i := 0; i < 3; i++ { + if frame.header.Rsv[i] { + j := uint(6 - i) + b |= 1 << j + } + } + b |= frame.header.OpCode + header = append(header, b) + if frame.header.MaskingKey != nil { + b = 0x80 + } else { + b = 0 + } + lengthFields := 0 + length := len(msg) + switch { + case length <= 125: + b |= byte(length) + case length < 65536: + b |= 126 + lengthFields = 2 + default: + b |= 127 + lengthFields = 8 + } + header = append(header, b) + for i := 0; i < lengthFields; i++ { + j := uint((lengthFields - i - 1) * 8) + b = byte((length >> j) & 0xff) + header = append(header, b) + } + if frame.header.MaskingKey != nil { + if len(frame.header.MaskingKey) != 4 { + return 0, ErrBadMaskingKey + } + header = append(header, frame.header.MaskingKey...) + frame.writer.Write(header) + var data []byte + + for i := 0; i < length; i++ { + data = append(data, msg[i]^frame.header.MaskingKey[i%4]) + } + frame.writer.Write(data) + err = frame.writer.Flush() + return length, err + } + frame.writer.Write(header) + frame.writer.Write(msg) + err = frame.writer.Flush() + return length, err +} + +func (frame *hybiFrameWriter) Close() os.Error { return nil } + +type hybiFrameWriterFactory struct { + *bufio.Writer + needMaskingKey bool +} + +func (buf hybiFrameWriterFactory) NewFrameWriter(payloadType byte) (frame frameWriter, err os.Error) { + frameHeader := &hybiFrameHeader{Fin: true, OpCode: payloadType} + if buf.needMaskingKey { + frameHeader.MaskingKey, err = generateMaskingKey() + if err != nil { + return nil, err + } + } + return &hybiFrameWriter{writer: buf.Writer, header: frameHeader}, nil +} + +type hybiFrameHandler struct { + conn *Conn + payloadType byte +} + +func (handler *hybiFrameHandler) HandleFrame(frame frameReader) (r frameReader, err os.Error) { + if handler.conn.IsServerConn() { + // The client MUST mask all frames sent to the server. + if frame.(*hybiFrameReader).header.MaskingKey == nil { + handler.WriteClose(closeStatusProtocolError) + return nil, os.EOF + } + } else { + // The server MUST NOT mask all frames. + if frame.(*hybiFrameReader).header.MaskingKey != nil { + handler.WriteClose(closeStatusProtocolError) + return nil, os.EOF + } + } + if header := frame.HeaderReader(); header != nil { + io.Copy(ioutil.Discard, header) + } + switch frame.PayloadType() { + case ContinuationFrame: + frame.(*hybiFrameReader).header.OpCode = handler.payloadType + case TextFrame, BinaryFrame: + handler.payloadType = frame.PayloadType() + case CloseFrame: + return nil, os.EOF + case PingFrame: + pingMsg := make([]byte, maxControlFramePayloadLength) + n, err := io.ReadFull(frame, pingMsg) + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + io.Copy(ioutil.Discard, frame) + n, err = handler.WritePong(pingMsg[:n]) + if err != nil { + return nil, err + } + return nil, nil + case PongFrame: + return nil, ErrNotImplemented + } + return frame, nil +} + +func (handler *hybiFrameHandler) WriteClose(status int) (err os.Error) { + handler.conn.wio.Lock() + defer handler.conn.wio.Unlock() + w, err := handler.conn.frameWriterFactory.NewFrameWriter(CloseFrame) + if err != nil { + return err + } + msg := make([]byte, 2) + binary.BigEndian.PutUint16(msg, uint16(status)) + _, err = w.Write(msg) + w.Close() + return err +} + +func (handler *hybiFrameHandler) WritePong(msg []byte) (n int, err os.Error) { + handler.conn.wio.Lock() + defer handler.conn.wio.Unlock() + w, err := handler.conn.frameWriterFactory.NewFrameWriter(PongFrame) + if err != nil { + return 0, err + } + n, err = w.Write(msg) + w.Close() + return n, err +} + +// newHybiConn creates a new WebSocket connection speaking hybi draft protocol. +func newHybiConn(config *Config, buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) *Conn { + if buf == nil { + br := bufio.NewReader(rwc) + bw := bufio.NewWriter(rwc) + buf = bufio.NewReadWriter(br, bw) + } + ws := &Conn{config: config, request: request, buf: buf, rwc: rwc, + frameReaderFactory: hybiFrameReaderFactory{buf.Reader}, + frameWriterFactory: hybiFrameWriterFactory{ + buf.Writer, request == nil}, + PayloadType: TextFrame, + defaultCloseStatus: closeStatusNormal} + ws.frameHandler = &hybiFrameHandler{conn: ws} + return ws +} + +// generateMaskingKey generates a masking key for a frame. +func generateMaskingKey() (maskingKey []byte, err os.Error) { + maskingKey = make([]byte, 4) + if _, err = io.ReadFull(rand.Reader, maskingKey); err != nil { + return + } + return +} + +// genetateNonce geneates a nonce consisting of a randomly selected 16-byte +// value that has been base64-encoded. +func generateNonce() (nonce []byte) { + key := make([]byte, 16) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + panic(err) + } + nonce = make([]byte, 24) + base64.StdEncoding.Encode(nonce, key) + return +} + +// getNonceAccept computes the base64-encoded SHA-1 of the concatenation of +// the nonce ("Sec-WebSocket-Key" value) with the websocket GUID string. +func getNonceAccept(nonce []byte) (expected []byte, err os.Error) { + h := sha1.New() + if _, err = h.Write(nonce); err != nil { + return + } + if _, err = h.Write([]byte(websocketGUID)); err != nil { + return + } + expected = make([]byte, 28) + base64.StdEncoding.Encode(expected, h.Sum()) + return +} + +func isHybiVersion(version int) bool { + switch version { + case ProtocolVersionHybi08, ProtocolVersionHybi13: + return true + default: + } + return false +} + +// Client handhake described in draft-ietf-hybi-thewebsocket-protocol-17 +func hybiClientHandshake(config *Config, br *bufio.Reader, bw *bufio.Writer) (err os.Error) { + if !isHybiVersion(config.Version) { + panic("wrong protocol version.") + } + + bw.WriteString("GET " + config.Location.RawPath + " HTTP/1.1\r\n") + + bw.WriteString("Host: " + config.Location.Host + "\r\n") + bw.WriteString("Upgrade: websocket\r\n") + bw.WriteString("Connection: Upgrade\r\n") + nonce := generateNonce() + if config.handshakeData != nil { + nonce = []byte(config.handshakeData["key"]) + } + bw.WriteString("Sec-WebSocket-Key: " + string(nonce) + "\r\n") + if config.Version == ProtocolVersionHybi13 { + bw.WriteString("Origin: " + strings.ToLower(config.Origin.String()) + "\r\n") + } else if config.Version == ProtocolVersionHybi08 { + bw.WriteString("Sec-WebSocket-Origin: " + strings.ToLower(config.Origin.String()) + "\r\n") + } + bw.WriteString("Sec-WebSocket-Version: " + fmt.Sprintf("%d", config.Version) + "\r\n") + if len(config.Protocol) > 0 { + bw.WriteString("Sec-WebSocket-Protocol: " + strings.Join(config.Protocol, ", ") + "\r\n") + } + // TODO(ukai): send extensions. + // TODO(ukai): send cookie if any. + + bw.WriteString("\r\n") + if err = bw.Flush(); err != nil { + return err + } + + resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) + if err != nil { + return err + } + if resp.StatusCode != 101 { + return ErrBadStatus + } + if strings.ToLower(resp.Header.Get("Upgrade")) != "websocket" || + strings.ToLower(resp.Header.Get("Connection")) != "upgrade" { + return ErrBadUpgrade + } + expectedAccept, err := getNonceAccept(nonce) + if err != nil { + return err + } + if resp.Header.Get("Sec-WebSocket-Accept") != string(expectedAccept) { + return ErrChallengeResponse + } + if resp.Header.Get("Sec-WebSocket-Extensions") != "" { + return ErrUnsupportedExtensions + } + offeredProtocol := resp.Header.Get("Sec-WebSocket-Protocol") + if offeredProtocol != "" { + protocolMatched := false + for i := 0; i < len(config.Protocol); i++ { + if config.Protocol[i] == offeredProtocol { + protocolMatched = true + break + } + } + if !protocolMatched { + return ErrBadWebSocketProtocol + } + config.Protocol = []string{offeredProtocol} + } + + return nil +} + +// newHybiClientConn creates a client WebSocket connection after handshake. +func newHybiClientConn(config *Config, buf *bufio.ReadWriter, rwc io.ReadWriteCloser) *Conn { + return newHybiConn(config, buf, rwc, nil) +} + +// A HybiServerHandshaker performs a server handshake using hybi draft protocol. +type hybiServerHandshaker struct { + *Config + accept []byte +} + +func (c *hybiServerHandshaker) ReadHandshake(buf *bufio.Reader, req *http.Request) (code int, err os.Error) { + c.Version = ProtocolVersionHybi13 + if req.Method != "GET" { + return http.StatusMethodNotAllowed, ErrBadRequestMethod + } + // HTTP version can be safely ignored. + + if strings.ToLower(req.Header.Get("Upgrade")) != "websocket" || + !strings.Contains(strings.ToLower(req.Header.Get("Connection")), "upgrade") { + return http.StatusBadRequest, ErrNotWebSocket + } + + key := req.Header.Get("Sec-Websocket-Key") + if key == "" { + return http.StatusBadRequest, ErrChallengeResponse + } + version := req.Header.Get("Sec-Websocket-Version") + var origin string + switch version { + case "13": + c.Version = ProtocolVersionHybi13 + origin = req.Header.Get("Origin") + case "8": + c.Version = ProtocolVersionHybi08 + origin = req.Header.Get("Sec-Websocket-Origin") + default: + return http.StatusBadRequest, ErrBadWebSocketVersion + } + c.Origin, err = url.ParseRequest(origin) + if err != nil { + return http.StatusForbidden, err + } + var scheme string + if req.TLS != nil { + scheme = "wss" + } else { + scheme = "ws" + } + c.Location, err = url.ParseRequest(scheme + "://" + req.Host + req.URL.RawPath) + if err != nil { + return http.StatusBadRequest, err + } + protocol := strings.TrimSpace(req.Header.Get("Sec-Websocket-Protocol")) + protocols := strings.Split(protocol, ",") + for i := 0; i < len(protocols); i++ { + c.Protocol = append(c.Protocol, strings.TrimSpace(protocols[i])) + } + c.accept, err = getNonceAccept([]byte(key)) + if err != nil { + return http.StatusInternalServerError, err + } + return http.StatusSwitchingProtocols, nil +} + +func (c *hybiServerHandshaker) AcceptHandshake(buf *bufio.Writer) (err os.Error) { + if len(c.Protocol) > 0 { + if len(c.Protocol) != 1 { + return ErrBadWebSocketProtocol + } + } + buf.WriteString("HTTP/1.1 101 Switching Protocols\r\n") + buf.WriteString("Upgrade: websocket\r\n") + buf.WriteString("Connection: Upgrade\r\n") + buf.WriteString("Sec-WebSocket-Accept: " + string(c.accept) + "\r\n") + if len(c.Protocol) > 0 { + buf.WriteString("Sec-WebSocket-Protocol: " + c.Protocol[0] + "\r\n") + } + // TODO(ukai): support extensions + buf.WriteString("\r\n") + return buf.Flush() +} + +func (c *hybiServerHandshaker) NewServerConn(buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) *Conn { + return newHybiServerConn(c.Config, buf, rwc, request) +} + +// newHybiServerConn returns a new WebSocket connection speaking hybi draft protocol. +func newHybiServerConn(config *Config, buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) *Conn { + return newHybiConn(config, buf, rwc, request) +} diff --git a/libgo/go/websocket/hybi_test.go b/libgo/go/websocket/hybi_test.go new file mode 100644 index 0000000..9db57e3 --- /dev/null +++ b/libgo/go/websocket/hybi_test.go @@ -0,0 +1,584 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "bytes" + "fmt" + "http" + "os" + "strings" + "testing" + "url" +) + +// Test the getNonceAccept function with values in +// http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-17 +func TestSecWebSocketAccept(t *testing.T) { + nonce := []byte("dGhlIHNhbXBsZSBub25jZQ==") + expected := []byte("s3pPLMBiTxaQ9kYGzzhZRbK+xOo=") + accept, err := getNonceAccept(nonce) + if err != nil { + t.Errorf("getNonceAccept: returned error %v", err) + return + } + if !bytes.Equal(expected, accept) { + t.Errorf("getNonceAccept: expected %q got %q", expected, accept) + } +} + +func TestHybiClientHandshake(t *testing.T) { + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + br := bufio.NewReader(strings.NewReader(`HTTP/1.1 101 Switching Protocols +Upgrade: websocket +Connection: Upgrade +Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo= +Sec-WebSocket-Protocol: chat + +`)) + var err os.Error + config := new(Config) + config.Location, err = url.ParseRequest("ws://server.example.com/chat") + if err != nil { + t.Fatal("location url", err) + } + config.Origin, err = url.ParseRequest("http://example.com") + if err != nil { + t.Fatal("origin url", err) + } + config.Protocol = append(config.Protocol, "chat") + config.Protocol = append(config.Protocol, "superchat") + config.Version = ProtocolVersionHybi13 + + config.handshakeData = map[string]string{ + "key": "dGhlIHNhbXBsZSBub25jZQ==", + } + err = hybiClientHandshake(config, br, bw) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + req, err := http.ReadRequest(bufio.NewReader(b)) + if err != nil { + t.Fatalf("read request: %v", err) + } + if req.Method != "GET" { + t.Errorf("request method expected GET, but got %q", req.Method) + } + if req.URL.Path != "/chat" { + t.Errorf("request path expected /chat, but got %q", req.URL.Path) + } + if req.Proto != "HTTP/1.1" { + t.Errorf("request proto expected HTTP/1.1, but got %q", req.Proto) + } + if req.Host != "server.example.com" { + t.Errorf("request Host expected server.example.com, but got %v", req.Host) + } + var expectedHeader = map[string]string{ + "Connection": "Upgrade", + "Upgrade": "websocket", + "Sec-Websocket-Key": config.handshakeData["key"], + "Origin": config.Origin.String(), + "Sec-Websocket-Protocol": "chat, superchat", + "Sec-Websocket-Version": fmt.Sprintf("%d", ProtocolVersionHybi13), + } + for k, v := range expectedHeader { + if req.Header.Get(k) != v { + t.Errorf(fmt.Sprintf("%s expected %q but got %q", k, v, req.Header.Get(k))) + } + } +} + +func TestHybiClientHandshakeHybi08(t *testing.T) { + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + br := bufio.NewReader(strings.NewReader(`HTTP/1.1 101 Switching Protocols +Upgrade: websocket +Connection: Upgrade +Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo= +Sec-WebSocket-Protocol: chat + +`)) + var err os.Error + config := new(Config) + config.Location, err = url.ParseRequest("ws://server.example.com/chat") + if err != nil { + t.Fatal("location url", err) + } + config.Origin, err = url.ParseRequest("http://example.com") + if err != nil { + t.Fatal("origin url", err) + } + config.Protocol = append(config.Protocol, "chat") + config.Protocol = append(config.Protocol, "superchat") + config.Version = ProtocolVersionHybi08 + + config.handshakeData = map[string]string{ + "key": "dGhlIHNhbXBsZSBub25jZQ==", + } + err = hybiClientHandshake(config, br, bw) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + req, err := http.ReadRequest(bufio.NewReader(b)) + if err != nil { + t.Fatalf("read request: %v", err) + } + if req.Method != "GET" { + t.Errorf("request method expected GET, but got %q", req.Method) + } + if req.URL.Path != "/chat" { + t.Errorf("request path expected /demo, but got %q", req.URL.Path) + } + if req.Proto != "HTTP/1.1" { + t.Errorf("request proto expected HTTP/1.1, but got %q", req.Proto) + } + if req.Host != "server.example.com" { + t.Errorf("request Host expected example.com, but got %v", req.Host) + } + var expectedHeader = map[string]string{ + "Connection": "Upgrade", + "Upgrade": "websocket", + "Sec-Websocket-Key": config.handshakeData["key"], + "Sec-Websocket-Origin": config.Origin.String(), + "Sec-Websocket-Protocol": "chat, superchat", + "Sec-Websocket-Version": fmt.Sprintf("%d", ProtocolVersionHybi08), + } + for k, v := range expectedHeader { + if req.Header.Get(k) != v { + t.Errorf(fmt.Sprintf("%s expected %q but got %q", k, v, req.Header.Get(k))) + } + } +} + +func TestHybiServerHandshake(t *testing.T) { + config := new(Config) + handshaker := &hybiServerHandshaker{Config: config} + br := bufio.NewReader(strings.NewReader(`GET /chat HTTP/1.1 +Host: server.example.com +Upgrade: websocket +Connection: Upgrade +Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ== +Origin: http://example.com +Sec-WebSocket-Protocol: chat, superchat +Sec-WebSocket-Version: 13 + +`)) + req, err := http.ReadRequest(br) + if err != nil { + t.Fatal("request", err) + } + code, err := handshaker.ReadHandshake(br, req) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + if code != http.StatusSwitchingProtocols { + t.Errorf("status expected %q but got %q", http.StatusSwitchingProtocols, code) + } + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + + config.Protocol = []string{"chat"} + + err = handshaker.AcceptHandshake(bw) + if err != nil { + t.Errorf("handshake response failed: %v", err) + } + expectedResponse := strings.Join([]string{ + "HTTP/1.1 101 Switching Protocols", + "Upgrade: websocket", + "Connection: Upgrade", + "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", + "Sec-WebSocket-Protocol: chat", + "", ""}, "\r\n") + + if b.String() != expectedResponse { + t.Errorf("handshake expected %q but got %q", expectedResponse, b.String()) + } +} + +func TestHybiServerHandshakeHybi08(t *testing.T) { + config := new(Config) + handshaker := &hybiServerHandshaker{Config: config} + br := bufio.NewReader(strings.NewReader(`GET /chat HTTP/1.1 +Host: server.example.com +Upgrade: websocket +Connection: Upgrade +Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ== +Sec-WebSocket-Origin: http://example.com +Sec-WebSocket-Protocol: chat, superchat +Sec-WebSocket-Version: 8 + +`)) + req, err := http.ReadRequest(br) + if err != nil { + t.Fatal("request", err) + } + code, err := handshaker.ReadHandshake(br, req) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + if code != http.StatusSwitchingProtocols { + t.Errorf("status expected %q but got %q", http.StatusSwitchingProtocols, code) + } + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + + config.Protocol = []string{"chat"} + + err = handshaker.AcceptHandshake(bw) + if err != nil { + t.Errorf("handshake response failed: %v", err) + } + expectedResponse := strings.Join([]string{ + "HTTP/1.1 101 Switching Protocols", + "Upgrade: websocket", + "Connection: Upgrade", + "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", + "Sec-WebSocket-Protocol: chat", + "", ""}, "\r\n") + + if b.String() != expectedResponse { + t.Errorf("handshake expected %q but got %q", expectedResponse, b.String()) + } +} + +func TestHybiServerHandshakeHybiBadVersion(t *testing.T) { + config := new(Config) + handshaker := &hybiServerHandshaker{Config: config} + br := bufio.NewReader(strings.NewReader(`GET /chat HTTP/1.1 +Host: server.example.com +Upgrade: websocket +Connection: Upgrade +Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ== +Sec-WebSocket-Origin: http://example.com +Sec-WebSocket-Protocol: chat, superchat +Sec-WebSocket-Version: 9 + +`)) + req, err := http.ReadRequest(br) + if err != nil { + t.Fatal("request", err) + } + code, err := handshaker.ReadHandshake(br, req) + if err != ErrBadWebSocketVersion { + t.Errorf("handshake expected err %q but got %q", ErrBadWebSocketVersion, err) + } + if code != http.StatusBadRequest { + t.Errorf("status expected %q but got %q", http.StatusBadRequest, code) + } +} + +func testHybiFrame(t *testing.T, testHeader, testPayload, testMaskedPayload []byte, frameHeader *hybiFrameHeader) { + b := bytes.NewBuffer([]byte{}) + frameWriterFactory := &hybiFrameWriterFactory{bufio.NewWriter(b), false} + w, _ := frameWriterFactory.NewFrameWriter(TextFrame) + w.(*hybiFrameWriter).header = frameHeader + _, err := w.Write(testPayload) + w.Close() + if err != nil { + t.Errorf("Write error %q", err) + } + var expectedFrame []byte + expectedFrame = append(expectedFrame, testHeader...) + expectedFrame = append(expectedFrame, testMaskedPayload...) + if !bytes.Equal(expectedFrame, b.Bytes()) { + t.Errorf("frame expected %q got %q", expectedFrame, b.Bytes()) + } + frameReaderFactory := &hybiFrameReaderFactory{bufio.NewReader(b)} + r, err := frameReaderFactory.NewFrameReader() + if err != nil { + t.Errorf("Read error %q", err) + } + if header := r.HeaderReader(); header == nil { + t.Errorf("no header") + } else { + actualHeader := make([]byte, r.Len()) + n, err := header.Read(actualHeader) + if err != nil { + t.Errorf("Read header error %q", err) + } else { + if n < len(testHeader) { + t.Errorf("header too short %q got %q", testHeader, actualHeader[:n]) + } + if !bytes.Equal(testHeader, actualHeader[:n]) { + t.Errorf("header expected %q got %q", testHeader, actualHeader[:n]) + } + } + } + if trailer := r.TrailerReader(); trailer != nil { + t.Errorf("unexpected trailer %q", trailer) + } + frame := r.(*hybiFrameReader) + if frameHeader.Fin != frame.header.Fin || + frameHeader.OpCode != frame.header.OpCode || + len(testPayload) != int(frame.header.Length) { + t.Errorf("mismatch %v (%d) vs %v", frameHeader, len(testPayload), frame) + } + payload := make([]byte, len(testPayload)) + _, err = r.Read(payload) + if err != nil { + t.Errorf("read %v", err) + } + if !bytes.Equal(testPayload, payload) { + t.Errorf("payload %q vs %q", testPayload, payload) + } +} + +func TestHybiShortTextFrame(t *testing.T) { + frameHeader := &hybiFrameHeader{Fin: true, OpCode: TextFrame} + payload := []byte("hello") + testHybiFrame(t, []byte{0x81, 0x05}, payload, payload, frameHeader) + + payload = make([]byte, 125) + testHybiFrame(t, []byte{0x81, 125}, payload, payload, frameHeader) +} + +func TestHybiShortMaskedTextFrame(t *testing.T) { + frameHeader := &hybiFrameHeader{Fin: true, OpCode: TextFrame, + MaskingKey: []byte{0xcc, 0x55, 0x80, 0x20}} + payload := []byte("hello") + maskedPayload := []byte{0xa4, 0x30, 0xec, 0x4c, 0xa3} + header := []byte{0x81, 0x85} + header = append(header, frameHeader.MaskingKey...) + testHybiFrame(t, header, payload, maskedPayload, frameHeader) +} + +func TestHybiShortBinaryFrame(t *testing.T) { + frameHeader := &hybiFrameHeader{Fin: true, OpCode: BinaryFrame} + payload := []byte("hello") + testHybiFrame(t, []byte{0x82, 0x05}, payload, payload, frameHeader) + + payload = make([]byte, 125) + testHybiFrame(t, []byte{0x82, 125}, payload, payload, frameHeader) +} + +func TestHybiControlFrame(t *testing.T) { + frameHeader := &hybiFrameHeader{Fin: true, OpCode: PingFrame} + payload := []byte("hello") + testHybiFrame(t, []byte{0x89, 0x05}, payload, payload, frameHeader) + + frameHeader = &hybiFrameHeader{Fin: true, OpCode: PongFrame} + testHybiFrame(t, []byte{0x8A, 0x05}, payload, payload, frameHeader) + + frameHeader = &hybiFrameHeader{Fin: true, OpCode: CloseFrame} + payload = []byte{0x03, 0xe8} // 1000 + testHybiFrame(t, []byte{0x88, 0x02}, payload, payload, frameHeader) +} + +func TestHybiLongFrame(t *testing.T) { + frameHeader := &hybiFrameHeader{Fin: true, OpCode: TextFrame} + payload := make([]byte, 126) + testHybiFrame(t, []byte{0x81, 126, 0x00, 126}, payload, payload, frameHeader) + + payload = make([]byte, 65535) + testHybiFrame(t, []byte{0x81, 126, 0xff, 0xff}, payload, payload, frameHeader) + + payload = make([]byte, 65536) + testHybiFrame(t, []byte{0x81, 127, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00}, payload, payload, frameHeader) +} + +func TestHybiClientRead(t *testing.T) { + wireData := []byte{0x81, 0x05, 'h', 'e', 'l', 'l', 'o', + 0x89, 0x05, 'h', 'e', 'l', 'l', 'o', // ping + 0x81, 0x05, 'w', 'o', 'r', 'l', 'd'} + br := bufio.NewReader(bytes.NewBuffer(wireData)) + bw := bufio.NewWriter(bytes.NewBuffer([]byte{})) + conn := newHybiConn(newConfig(t, "/"), bufio.NewReadWriter(br, bw), nil, nil) + + msg := make([]byte, 512) + n, err := conn.Read(msg) + if err != nil { + t.Errorf("read 1st frame, error %q", err) + } + if n != 5 { + t.Errorf("read 1st frame, expect 5, got %d", n) + } + if !bytes.Equal(wireData[2:7], msg[:n]) { + t.Errorf("read 1st frame %v, got %v", wireData[2:7], msg[:n]) + } + n, err = conn.Read(msg) + if err != nil { + t.Errorf("read 2nd frame, error %q", err) + } + if n != 5 { + t.Errorf("read 2nd frame, expect 5, got %d", n) + } + if !bytes.Equal(wireData[16:21], msg[:n]) { + t.Errorf("read 2nd frame %v, got %v", wireData[16:21], msg[:n]) + } + n, err = conn.Read(msg) + if err == nil { + t.Errorf("read not EOF") + } + if n != 0 { + t.Errorf("expect read 0, got %d", n) + } +} + +func TestHybiShortRead(t *testing.T) { + wireData := []byte{0x81, 0x05, 'h', 'e', 'l', 'l', 'o', + 0x89, 0x05, 'h', 'e', 'l', 'l', 'o', // ping + 0x81, 0x05, 'w', 'o', 'r', 'l', 'd'} + br := bufio.NewReader(bytes.NewBuffer(wireData)) + bw := bufio.NewWriter(bytes.NewBuffer([]byte{})) + conn := newHybiConn(newConfig(t, "/"), bufio.NewReadWriter(br, bw), nil, nil) + + step := 0 + pos := 0 + expectedPos := []int{2, 5, 16, 19} + expectedLen := []int{3, 2, 3, 2} + for { + msg := make([]byte, 3) + n, err := conn.Read(msg) + if step >= len(expectedPos) { + if err == nil { + t.Errorf("read not EOF") + } + if n != 0 { + t.Errorf("expect read 0, got %d", n) + } + return + } + pos = expectedPos[step] + endPos := pos + expectedLen[step] + if err != nil { + t.Errorf("read from %d, got error %q", pos, err) + return + } + if n != endPos-pos { + t.Errorf("read from %d, expect %d, got %d", pos, endPos-pos, n) + } + if !bytes.Equal(wireData[pos:endPos], msg[:n]) { + t.Errorf("read from %d, frame %v, got %v", pos, wireData[pos:endPos], msg[:n]) + } + step++ + } +} + +func TestHybiServerRead(t *testing.T) { + wireData := []byte{0x81, 0x85, 0xcc, 0x55, 0x80, 0x20, + 0xa4, 0x30, 0xec, 0x4c, 0xa3, // hello + 0x89, 0x85, 0xcc, 0x55, 0x80, 0x20, + 0xa4, 0x30, 0xec, 0x4c, 0xa3, // ping: hello + 0x81, 0x85, 0xed, 0x83, 0xb4, 0x24, + 0x9a, 0xec, 0xc6, 0x48, 0x89, // world + } + br := bufio.NewReader(bytes.NewBuffer(wireData)) + bw := bufio.NewWriter(bytes.NewBuffer([]byte{})) + conn := newHybiConn(newConfig(t, "/"), bufio.NewReadWriter(br, bw), nil, new(http.Request)) + + expected := [][]byte{[]byte("hello"), []byte("world")} + + msg := make([]byte, 512) + n, err := conn.Read(msg) + if err != nil { + t.Errorf("read 1st frame, error %q", err) + } + if n != 5 { + t.Errorf("read 1st frame, expect 5, got %d", n) + } + if !bytes.Equal(expected[0], msg[:n]) { + t.Errorf("read 1st frame %q, got %q", expected[0], msg[:n]) + } + + n, err = conn.Read(msg) + if err != nil { + t.Errorf("read 2nd frame, error %q", err) + } + if n != 5 { + t.Errorf("read 2nd frame, expect 5, got %d", n) + } + if !bytes.Equal(expected[1], msg[:n]) { + t.Errorf("read 2nd frame %q, got %q", expected[1], msg[:n]) + } + + n, err = conn.Read(msg) + if err == nil { + t.Errorf("read not EOF") + } + if n != 0 { + t.Errorf("expect read 0, got %d", n) + } +} + +func TestHybiServerReadWithoutMasking(t *testing.T) { + wireData := []byte{0x81, 0x05, 'h', 'e', 'l', 'l', 'o'} + br := bufio.NewReader(bytes.NewBuffer(wireData)) + bw := bufio.NewWriter(bytes.NewBuffer([]byte{})) + conn := newHybiConn(newConfig(t, "/"), bufio.NewReadWriter(br, bw), nil, new(http.Request)) + // server MUST close the connection upon receiving a non-masked frame. + msg := make([]byte, 512) + _, err := conn.Read(msg) + if err != os.EOF { + t.Errorf("read 1st frame, expect %q, but got %q", os.EOF, err) + } +} + +func TestHybiClientReadWithMasking(t *testing.T) { + wireData := []byte{0x81, 0x85, 0xcc, 0x55, 0x80, 0x20, + 0xa4, 0x30, 0xec, 0x4c, 0xa3, // hello + } + br := bufio.NewReader(bytes.NewBuffer(wireData)) + bw := bufio.NewWriter(bytes.NewBuffer([]byte{})) + conn := newHybiConn(newConfig(t, "/"), bufio.NewReadWriter(br, bw), nil, nil) + + // client MUST close the connection upon receiving a masked frame. + msg := make([]byte, 512) + _, err := conn.Read(msg) + if err != os.EOF { + t.Errorf("read 1st frame, expect %q, but got %q", os.EOF, err) + } +} + +// Test the hybiServerHandshaker supports firefox implementation and +// checks Connection request header include (but it's not necessary +// equal to) "upgrade" +func TestHybiServerFirefoxHandshake(t *testing.T) { + config := new(Config) + handshaker := &hybiServerHandshaker{Config: config} + br := bufio.NewReader(strings.NewReader(`GET /chat HTTP/1.1 +Host: server.example.com +Upgrade: websocket +Connection: keep-alive, upgrade +Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ== +Origin: http://example.com +Sec-WebSocket-Protocol: chat, superchat +Sec-WebSocket-Version: 13 + +`)) + req, err := http.ReadRequest(br) + if err != nil { + t.Fatal("request", err) + } + code, err := handshaker.ReadHandshake(br, req) + if err != nil { + t.Errorf("handshake failed: %v", err) + } + if code != http.StatusSwitchingProtocols { + t.Errorf("status expected %q but got %q", http.StatusSwitchingProtocols, code) + } + b := bytes.NewBuffer([]byte{}) + bw := bufio.NewWriter(b) + + config.Protocol = []string{"chat"} + + err = handshaker.AcceptHandshake(bw) + if err != nil { + t.Errorf("handshake response failed: %v", err) + } + expectedResponse := strings.Join([]string{ + "HTTP/1.1 101 Switching Protocols", + "Upgrade: websocket", + "Connection: Upgrade", + "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", + "Sec-WebSocket-Protocol: chat", + "", ""}, "\r\n") + + if b.String() != expectedResponse { + t.Errorf("handshake expected %q but got %q", expectedResponse, b.String()) + } +} diff --git a/libgo/go/websocket/server.go b/libgo/go/websocket/server.go index e0e7c87..a1d1d48 100644 --- a/libgo/go/websocket/server.go +++ b/libgo/go/websocket/server.go @@ -5,11 +5,48 @@ package websocket import ( + "bufio" + "fmt" "http" "io" - "strings" + "os" ) +func newServerConn(rwc io.ReadWriteCloser, buf *bufio.ReadWriter, req *http.Request) (conn *Conn, err os.Error) { + config := new(Config) + var hs serverHandshaker = &hybiServerHandshaker{Config: config} + code, err := hs.ReadHandshake(buf.Reader, req) + if err == ErrBadWebSocketVersion { + fmt.Fprintf(buf, "HTTP/1.1 %03d %s\r\n", code, http.StatusText(code)) + fmt.Fprintf(buf, "Sec-WebSocket-Version: %s\r\n", SupportedProtocolVersion) + buf.WriteString("\r\n") + buf.WriteString(err.String()) + return + } + if err != nil { + hs = &hixie76ServerHandshaker{Config: config} + code, err = hs.ReadHandshake(buf.Reader, req) + } + if err != nil { + hs = &hixie75ServerHandshaker{Config: config} + code, err = hs.ReadHandshake(buf.Reader, req) + } + if err != nil { + fmt.Fprintf(buf, "HTTP/1.1 %03d %s\r\n", code, http.StatusText(code)) + buf.WriteString("\r\n") + buf.WriteString(err.String()) + return + } + config.Protocol = nil + + err = hs.AcceptHandshake(buf.Writer) + if err != nil { + return + } + conn = hs.NewServerConn(buf, rwc, req) + return +} + /* Handler is an interface to a WebSocket. @@ -23,7 +60,7 @@ A trivial example server: "websocket" ) - // Echo the data received on the Web Socket. + // Echo the data received on the WebSocket. func EchoServer(ws *websocket.Conn) { io.Copy(ws, ws); } @@ -38,26 +75,8 @@ A trivial example server: */ type Handler func(*Conn) -/* -Gets key number from Sec-WebSocket-Key<n>: field as described -in 5.2 Sending the server's opening handshake, 4. -*/ -func getKeyNumber(s string) (r uint32) { - // 4. Let /key-number_n/ be the digits (characters in the range - // U+0030 DIGIT ZERO (0) to U+0039 DIGIT NINE (9)) in /key_1/, - // interpreted as a base ten integer, ignoring all other characters - // in /key_n/. - r = 0 - for i := 0; i < len(s); i++ { - if s[i] >= '0' && s[i] <= '9' { - r = r*10 + uint32(s[i]) - '0' - } - } - return -} - // ServeHTTP implements the http.Handler interface for a Web Socket -func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { rwc, buf, err := w.(http.Hijacker).Hijack() if err != nil { panic("Hijack failed: " + err.String()) @@ -67,153 +86,12 @@ func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { // the client did not send a handshake that matches with protocol // specification. defer rwc.Close() - - if req.Method != "GET" { - return - } - // HTTP version can be safely ignored. - - if strings.ToLower(req.Header.Get("Upgrade")) != "websocket" || - strings.ToLower(req.Header.Get("Connection")) != "upgrade" { - return - } - - // TODO(ukai): check Host - origin := req.Header.Get("Origin") - if origin == "" { - return - } - - key1 := req.Header.Get("Sec-Websocket-Key1") - if key1 == "" { - return - } - key2 := req.Header.Get("Sec-Websocket-Key2") - if key2 == "" { - return - } - key3 := make([]byte, 8) - if _, err := io.ReadFull(buf, key3); err != nil { - return - } - - var location string - if req.TLS != nil { - location = "wss://" + req.Host + req.URL.RawPath - } else { - location = "ws://" + req.Host + req.URL.RawPath - } - - // Step 4. get key number in Sec-WebSocket-Key<n> fields. - keyNumber1 := getKeyNumber(key1) - keyNumber2 := getKeyNumber(key2) - - // Step 5. get number of spaces in Sec-WebSocket-Key<n> fields. - space1 := uint32(strings.Count(key1, " ")) - space2 := uint32(strings.Count(key2, " ")) - if space1 == 0 || space2 == 0 { - return - } - - // Step 6. key number must be an integral multiple of spaces. - if keyNumber1%space1 != 0 || keyNumber2%space2 != 0 { - return - } - - // Step 7. let part be key number divided by spaces. - part1 := keyNumber1 / space1 - part2 := keyNumber2 / space2 - - // Step 8. let challenge be concatenation of part1, part2 and key3. - // Step 9. get MD5 fingerprint of challenge. - response, err := getChallengeResponse(part1, part2, key3) - if err != nil { - return - } - - // Step 10. send response status line. - buf.WriteString("HTTP/1.1 101 WebSocket Protocol Handshake\r\n") - // Step 11. send response headers. - buf.WriteString("Upgrade: WebSocket\r\n") - buf.WriteString("Connection: Upgrade\r\n") - buf.WriteString("Sec-WebSocket-Location: " + location + "\r\n") - buf.WriteString("Sec-WebSocket-Origin: " + origin + "\r\n") - protocol := strings.TrimSpace(req.Header.Get("Sec-Websocket-Protocol")) - if protocol != "" { - buf.WriteString("Sec-WebSocket-Protocol: " + protocol + "\r\n") - } - // Step 12. send CRLF. - buf.WriteString("\r\n") - // Step 13. send response data. - buf.Write(response) - if err := buf.Flush(); err != nil { - return - } - ws := newConn(origin, location, protocol, buf, rwc) - ws.Request = req - f(ws) -} - -/* -Draft75Handler is an interface to a WebSocket based on the -(soon obsolete) draft-hixie-thewebsocketprotocol-75. -*/ -type Draft75Handler func(*Conn) - -// ServeHTTP implements the http.Handler interface for a Web Socket. -func (f Draft75Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if req.Method != "GET" || req.Proto != "HTTP/1.1" { - w.WriteHeader(http.StatusBadRequest) - io.WriteString(w, "Unexpected request") - return - } - if req.Header.Get("Upgrade") != "WebSocket" { - w.WriteHeader(http.StatusBadRequest) - io.WriteString(w, "missing Upgrade: WebSocket header") - return - } - if req.Header.Get("Connection") != "Upgrade" { - w.WriteHeader(http.StatusBadRequest) - io.WriteString(w, "missing Connection: Upgrade header") - return - } - origin := strings.TrimSpace(req.Header.Get("Origin")) - if origin == "" { - w.WriteHeader(http.StatusBadRequest) - io.WriteString(w, "missing Origin header") - return - } - - rwc, buf, err := w.(http.Hijacker).Hijack() + conn, err := newServerConn(rwc, buf, req) if err != nil { - panic("Hijack failed: " + err.String()) return } - defer rwc.Close() - - var location string - if req.TLS != nil { - location = "wss://" + req.Host + req.URL.RawPath - } else { - location = "ws://" + req.Host + req.URL.RawPath - } - - // TODO(ukai): verify origin,location,protocol. - - buf.WriteString("HTTP/1.1 101 Web Socket Protocol Handshake\r\n") - buf.WriteString("Upgrade: WebSocket\r\n") - buf.WriteString("Connection: Upgrade\r\n") - buf.WriteString("WebSocket-Origin: " + origin + "\r\n") - buf.WriteString("WebSocket-Location: " + location + "\r\n") - protocol := strings.TrimSpace(req.Header.Get("Websocket-Protocol")) - // canonical header key of WebSocket-Protocol. - if protocol != "" { - buf.WriteString("WebSocket-Protocol: " + protocol + "\r\n") - } - buf.WriteString("\r\n") - if err := buf.Flush(); err != nil { - return + if conn == nil { + panic("unepxected nil conn") } - ws := newConn(origin, location, protocol, buf, rwc) - f(ws) + h(conn) } diff --git a/libgo/go/websocket/websocket.go b/libgo/go/websocket/websocket.go index 7447cf8..a3750dd 100644 --- a/libgo/go/websocket/websocket.go +++ b/libgo/go/websocket/websocket.go @@ -2,145 +2,246 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package websocket implements a client and server for the Web Socket protocol. -// The protocol is defined at http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol +// Package websocket implements a client and server for the WebSocket protocol. +// The protocol is defined at http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol package websocket -// TODO(ukai): -// better logging. - import ( "bufio" - "crypto/md5" - "encoding/binary" + "crypto/tls" "http" "io" + "io/ioutil" + "json" "net" "os" + "sync" + "url" ) -// WebSocketAddr is an implementation of net.Addr for Web Sockets. -type WebSocketAddr string +const ( + ProtocolVersionHixie75 = -75 + ProtocolVersionHixie76 = -76 + ProtocolVersionHybi00 = 0 + ProtocolVersionHybi08 = 8 + ProtocolVersionHybi13 = 13 + ProtocolVersionHybi = ProtocolVersionHybi13 + SupportedProtocolVersion = "13, 8" + + ContinuationFrame = 0 + TextFrame = 1 + BinaryFrame = 2 + CloseFrame = 8 + PingFrame = 9 + PongFrame = 10 + UnknownFrame = 255 +) -// Network returns the network type for a Web Socket, "websocket". -func (addr WebSocketAddr) Network() string { return "websocket" } +// WebSocket protocol errors. +type ProtocolError struct { + ErrorString string +} -// String returns the network address for a Web Socket. -func (addr WebSocketAddr) String() string { return string(addr) } +func (err *ProtocolError) String() string { return err.ErrorString } -const ( - stateFrameByte = iota - stateFrameLength - stateFrameData - stateFrameTextData +var ( + ErrBadProtocolVersion = &ProtocolError{"bad protocol version"} + ErrBadScheme = &ProtocolError{"bad scheme"} + ErrBadStatus = &ProtocolError{"bad status"} + ErrBadUpgrade = &ProtocolError{"missing or bad upgrade"} + ErrBadWebSocketOrigin = &ProtocolError{"missing or bad WebSocket-Origin"} + ErrBadWebSocketLocation = &ProtocolError{"missing or bad WebSocket-Location"} + ErrBadWebSocketProtocol = &ProtocolError{"missing or bad WebSocket-Protocol"} + ErrBadWebSocketVersion = &ProtocolError{"missing or bad WebSocket Version"} + ErrChallengeResponse = &ProtocolError{"mismatch challenge/response"} + ErrBadFrame = &ProtocolError{"bad frame"} + ErrBadFrameBoundary = &ProtocolError{"not on frame boundary"} + ErrNotWebSocket = &ProtocolError{"not websocket protocol"} + ErrBadRequestMethod = &ProtocolError{"bad method"} + ErrNotSupported = &ProtocolError{"not supported"} ) -// Conn is a channel to communicate to a Web Socket. -// It implements the net.Conn interface. +// Addr is an implementation of net.Addr for WebSocket. +type Addr struct { + *url.URL +} + +// Network returns the network type for a WebSocket, "websocket". +func (addr *Addr) Network() string { return "websocket" } + +// Config is a WebSocket configuration +type Config struct { + // A WebSocket server address. + Location *url.URL + + // A Websocket client origin. + Origin *url.URL + + // WebSocket subprotocols. + Protocol []string + + // WebSocket protocol version. + Version int + + // TLS config for secure WebSocket (wss). + TlsConfig *tls.Config + + handshakeData map[string]string +} + +// serverHandshaker is an interface to handle WebSocket server side handshake. +type serverHandshaker interface { + // ReadHandshake reads handshake request message from client. + // Returns http response code and error if any. + ReadHandshake(buf *bufio.Reader, req *http.Request) (code int, err os.Error) + + // AcceptHandshake accepts the client handshake request and sends + // handshake response back to client. + AcceptHandshake(buf *bufio.Writer) (err os.Error) + + // NewServerConn creates a new WebSocket connection. + NewServerConn(buf *bufio.ReadWriter, rwc io.ReadWriteCloser, request *http.Request) (conn *Conn) +} + +// frameReader is an interface to read a WebSocket frame. +type frameReader interface { + // Reader is to read payload of the frame. + io.Reader + + // PayloadType returns payload type. + PayloadType() byte + + // HeaderReader returns a reader to read header of the frame. + HeaderReader() io.Reader + + // TrailerReader returns a reader to read trailer of the frame. + // If it returns nil, there is no trailer in the frame. + TrailerReader() io.Reader + + // Len returns total length of the frame, including header and trailer. + Len() int +} + +// frameReaderFactory is an interface to creates new frame reader. +type frameReaderFactory interface { + NewFrameReader() (r frameReader, err os.Error) +} + +// frameWriter is an interface to write a WebSocket frame. +type frameWriter interface { + // Writer is to write playload of the frame. + io.WriteCloser +} + +// frameWriterFactory is an interface to create new frame writer. +type frameWriterFactory interface { + NewFrameWriter(payloadType byte) (w frameWriter, err os.Error) +} + +type frameHandler interface { + HandleFrame(frame frameReader) (r frameReader, err os.Error) + WriteClose(status int) (err os.Error) +} + +// Conn represents a WebSocket connection. type Conn struct { - // The origin URI for the Web Socket. - Origin string - // The location URI for the Web Socket. - Location string - // The subprotocol for the Web Socket. - Protocol string - // The initial http Request (for the Server side only). - Request *http.Request + config *Config + request *http.Request buf *bufio.ReadWriter rwc io.ReadWriteCloser - // It holds text data in previous Read() that failed with small buffer. - data []byte - reading bool -} + rio sync.Mutex + frameReaderFactory + frameReader -// newConn creates a new Web Socket. -func newConn(origin, location, protocol string, buf *bufio.ReadWriter, rwc io.ReadWriteCloser) *Conn { - if buf == nil { - br := bufio.NewReader(rwc) - bw := bufio.NewWriter(rwc) - buf = bufio.NewReadWriter(br, bw) - } - ws := &Conn{Origin: origin, Location: location, Protocol: protocol, buf: buf, rwc: rwc} - return ws + wio sync.Mutex + frameWriterFactory + + frameHandler + PayloadType byte + defaultCloseStatus int } -// Read implements the io.Reader interface for a Conn. +// Read implements the io.Reader interface: +// it reads data of a frame from the WebSocket connection. +// if msg is not large enough for the frame data, it fills the msg and next Read +// will read the rest of the frame data. +// it reads Text frame or Binary frame. func (ws *Conn) Read(msg []byte) (n int, err os.Error) { -Frame: - for !ws.reading && len(ws.data) == 0 { - // Beginning of frame, possibly. - b, err := ws.buf.ReadByte() + ws.rio.Lock() + defer ws.rio.Unlock() +again: + if ws.frameReader == nil { + frame, err := ws.frameReaderFactory.NewFrameReader() if err != nil { return 0, err } - if b&0x80 == 0x80 { - // Skip length frame. - length := 0 - for { - c, err := ws.buf.ReadByte() - if err != nil { - return 0, err - } - length = length*128 + int(c&0x7f) - if c&0x80 == 0 { - break - } - } - for length > 0 { - _, err := ws.buf.ReadByte() - if err != nil { - return 0, err - } - } - continue Frame + ws.frameReader, err = ws.frameHandler.HandleFrame(frame) + if err != nil { + return 0, err } - // In text mode - if b != 0 { - // Skip this frame - for { - c, err := ws.buf.ReadByte() - if err != nil { - return 0, err - } - if c == '\xff' { - break - } - } - continue Frame + if ws.frameReader == nil { + goto again } - ws.reading = true } - if len(ws.data) == 0 { - ws.data, err = ws.buf.ReadSlice('\xff') - if err == nil { - ws.reading = false - ws.data = ws.data[:len(ws.data)-1] // trim \xff + n, err = ws.frameReader.Read(msg) + if err == os.EOF { + if trailer := ws.frameReader.TrailerReader(); trailer != nil { + io.Copy(ioutil.Discard, trailer) } + ws.frameReader = nil + goto again } - n = copy(msg, ws.data) - ws.data = ws.data[n:] return n, err } -// Write implements the io.Writer interface for a Conn. +// Write implements the io.Writer interface: +// it writes data as a frame to the WebSocket connection. func (ws *Conn) Write(msg []byte) (n int, err os.Error) { - ws.buf.WriteByte(0) - ws.buf.Write(msg) - ws.buf.WriteByte(0xff) - err = ws.buf.Flush() - return len(msg), err + ws.wio.Lock() + defer ws.wio.Unlock() + w, err := ws.frameWriterFactory.NewFrameWriter(ws.PayloadType) + if err != nil { + return 0, err + } + n, err = w.Write(msg) + w.Close() + if err != nil { + return n, err + } + return n, err +} + +// Close implements the io.Closer interface. +func (ws *Conn) Close() os.Error { + err := ws.frameHandler.WriteClose(ws.defaultCloseStatus) + if err != nil { + return err + } + return ws.rwc.Close() } -// Close implements the io.Closer interface for a Conn. -func (ws *Conn) Close() os.Error { return ws.rwc.Close() } +func (ws *Conn) IsClientConn() bool { return ws.request == nil } +func (ws *Conn) IsServerConn() bool { return ws.request != nil } -// LocalAddr returns the WebSocket Origin for the connection. -func (ws *Conn) LocalAddr() net.Addr { return WebSocketAddr(ws.Origin) } +// LocalAddr returns the WebSocket Origin for the connection for client, or +// the WebSocket location for server. +func (ws *Conn) LocalAddr() net.Addr { + if ws.IsClientConn() { + return &Addr{ws.config.Origin} + } + return &Addr{ws.config.Location} +} -// RemoteAddr returns the WebSocket locations for the connection. -func (ws *Conn) RemoteAddr() net.Addr { return WebSocketAddr(ws.Location) } +// RemoteAddr returns the WebSocket location for the connection for client, or +// the Websocket Origin for server. +func (ws *Conn) RemoteAddr() net.Addr { + if ws.IsClientConn() { + return &Addr{ws.config.Location} + } + return &Addr{ws.config.Origin} +} // SetTimeout sets the connection's network timeout in nanoseconds. func (ws *Conn) SetTimeout(nsec int64) os.Error { @@ -166,27 +267,143 @@ func (ws *Conn) SetWriteTimeout(nsec int64) os.Error { return os.EINVAL } -// getChallengeResponse computes the expected response from the -// challenge as described in section 5.1 Opening Handshake steps 42 to -// 43 of http://www.whatwg.org/specs/web-socket-protocol/ -func getChallengeResponse(number1, number2 uint32, key3 []byte) (expected []byte, err os.Error) { - // 41. Let /challenge/ be the concatenation of /number_1/, expressed - // a big-endian 32 bit integer, /number_2/, expressed in a big- - // endian 32 bit integer, and the eight bytes of /key_3/ in the - // order they were sent to the wire. - challenge := make([]byte, 16) - binary.BigEndian.PutUint32(challenge[0:], number1) - binary.BigEndian.PutUint32(challenge[4:], number2) - copy(challenge[8:], key3) +// Config returns the WebSocket config. +func (ws *Conn) Config() *Config { return ws.config } + +// Request returns the http request upgraded to the WebSocket. +// It is nil for client side. +func (ws *Conn) Request() *http.Request { return ws.request } + +// Codec represents a symmetric pair of functions that implement a codec. +type Codec struct { + Marshal func(v interface{}) (data []byte, payloadType byte, err os.Error) + Unmarshal func(data []byte, payloadType byte, v interface{}) (err os.Error) +} + +// Send sends v marshaled by cd.Marshal as single frame to ws. +func (cd Codec) Send(ws *Conn, v interface{}) (err os.Error) { + if err != nil { + return err + } + data, payloadType, err := cd.Marshal(v) + if err != nil { + return err + } + ws.wio.Lock() + defer ws.wio.Unlock() + w, err := ws.frameWriterFactory.NewFrameWriter(payloadType) + _, err = w.Write(data) + w.Close() + return err +} + +// Receive receives single frame from ws, unmarshaled by cd.Unmarshal and stores in v. +func (cd Codec) Receive(ws *Conn, v interface{}) (err os.Error) { + ws.rio.Lock() + defer ws.rio.Unlock() + if ws.frameReader != nil { + _, err = io.Copy(ioutil.Discard, ws.frameReader) + if err != nil { + return err + } + ws.frameReader = nil + } +again: + frame, err := ws.frameReaderFactory.NewFrameReader() + if err != nil { + return err + } + frame, err = ws.frameHandler.HandleFrame(frame) + if err != nil { + return err + } + if frame == nil { + goto again + } + payloadType := frame.PayloadType() + data, err := ioutil.ReadAll(frame) + if err != nil { + return err + } + return cd.Unmarshal(data, payloadType, v) +} - // 42. Let /expected/ be the MD5 fingerprint of /challenge/ as a big- - // endian 128 bit string. - h := md5.New() - if _, err = h.Write(challenge); err != nil { - return +func marshal(v interface{}) (msg []byte, payloadType byte, err os.Error) { + switch data := v.(type) { + case string: + return []byte(data), TextFrame, nil + case []byte: + return data, BinaryFrame, nil } - expected = h.Sum() - return + return nil, UnknownFrame, ErrNotSupported } -var _ net.Conn = (*Conn)(nil) // compile-time check that *Conn implements net.Conn. +func unmarshal(msg []byte, payloadType byte, v interface{}) (err os.Error) { + switch data := v.(type) { + case *string: + *data = string(msg) + return nil + case *[]byte: + *data = msg + return nil + } + return ErrNotSupported +} + +/* +Message is a codec to send/receive text/binary data in a frame on WebSocket connection. +To send/receive text frame, use string type. +To send/receive binary frame, use []byte type. + +Trivial usage: + + import "websocket" + + // receive text frame + var message string + websocket.Message.Receive(ws, &message) + + // send text frame + message = "hello" + websocket.Message.Send(ws, message) + + // receive binary frame + var data []byte + websocket.Message.Receive(ws, &data) + + // send binary frame + data = []byte{0, 1, 2} + websocket.Message.Send(ws, data) + +*/ +var Message = Codec{marshal, unmarshal} + +func jsonMarshal(v interface{}) (msg []byte, payloadType byte, err os.Error) { + msg, err = json.Marshal(v) + return msg, TextFrame, err +} + +func jsonUnmarshal(msg []byte, payloadType byte, v interface{}) (err os.Error) { + return json.Unmarshal(msg, v) +} + +/* +JSON is a codec to send/receive JSON data in a frame from a WebSocket connection. + +Trival usage: + + import "websocket" + + type T struct { + Msg string + Count int + } + + // receive JSON type T + var data T + websocket.JSON.Receive(ws, &data) + + // send JSON type T + websocket.JSON.Send(ws, data) +*/ +var JSON = Codec{jsonMarshal, jsonUnmarshal} diff --git a/libgo/go/websocket/websocket_test.go b/libgo/go/websocket/websocket_test.go index 71c3c85..240af4e 100644 --- a/libgo/go/websocket/websocket_test.go +++ b/libgo/go/websocket/websocket_test.go @@ -5,7 +5,6 @@ package websocket import ( - "bufio" "bytes" "fmt" "http" @@ -13,6 +12,7 @@ import ( "io" "log" "net" + "strings" "sync" "testing" "url" @@ -23,31 +23,38 @@ var once sync.Once func echoServer(ws *Conn) { io.Copy(ws, ws) } +type Count struct { + S string + N int +} + +func countServer(ws *Conn) { + for { + var count Count + err := JSON.Receive(ws, &count) + if err != nil { + return + } + count.N++ + count.S = strings.Repeat(count.S, count.N) + err = JSON.Send(ws, count) + if err != nil { + return + } + } +} + func startServer() { http.Handle("/echo", Handler(echoServer)) - http.Handle("/echoDraft75", Draft75Handler(echoServer)) + http.Handle("/count", Handler(countServer)) server := httptest.NewServer(nil) serverAddr = server.Listener.Addr().String() log.Print("Test WebSocket server listening on ", serverAddr) } -// Test the getChallengeResponse function with values from section -// 5.1 of the specification steps 18, 26, and 43 from -// http://www.whatwg.org/specs/web-socket-protocol/ -func TestChallenge(t *testing.T) { - var part1 uint32 = 777007543 - var part2 uint32 = 114997259 - key3 := []byte{0x47, 0x30, 0x22, 0x2D, 0x5A, 0x3F, 0x47, 0x58} - expected := []byte("0st3Rl&q-2ZU^weu") - - response, err := getChallengeResponse(part1, part2, key3) - if err != nil { - t.Errorf("getChallengeResponse: returned error %v", err) - return - } - if !bytes.Equal(expected, response) { - t.Errorf("getChallengeResponse: expected %q got %q", expected, response) - } +func newConfig(t *testing.T, path string) *Config { + config, _ := NewConfig(fmt.Sprintf("ws://%s%s", serverAddr, path), "http://localhost") + return config } func TestEcho(t *testing.T) { @@ -58,19 +65,18 @@ func TestEcho(t *testing.T) { if err != nil { t.Fatal("dialing", err) } - ws, err := newClient("/echo", "localhost", "http://localhost", - "ws://localhost/echo", "", client, handshake) + conn, err := NewClient(newConfig(t, "/echo"), client) if err != nil { t.Errorf("WebSocket handshake error: %v", err) return } msg := []byte("hello, world\n") - if _, err := ws.Write(msg); err != nil { + if _, err := conn.Write(msg); err != nil { t.Errorf("Write: %v", err) } var actual_msg = make([]byte, 512) - n, err := ws.Read(actual_msg) + n, err := conn.Read(actual_msg) if err != nil { t.Errorf("Read: %v", err) } @@ -78,10 +84,10 @@ func TestEcho(t *testing.T) { if !bytes.Equal(msg, actual_msg) { t.Errorf("Echo: expected %q got %q", msg, actual_msg) } - ws.Close() + conn.Close() } -func TestEchoDraft75(t *testing.T) { +func TestAddr(t *testing.T) { once.Do(startServer) // websocket.Dial() @@ -89,27 +95,64 @@ func TestEchoDraft75(t *testing.T) { if err != nil { t.Fatal("dialing", err) } - ws, err := newClient("/echoDraft75", "localhost", "http://localhost", - "ws://localhost/echoDraft75", "", client, draft75handshake) + conn, err := NewClient(newConfig(t, "/echo"), client) if err != nil { - t.Errorf("WebSocket handshake: %v", err) + t.Errorf("WebSocket handshake error: %v", err) return } - msg := []byte("hello, world\n") - if _, err := ws.Write(msg); err != nil { - t.Errorf("Write: error %v", err) + ra := conn.RemoteAddr().String() + if !strings.HasPrefix(ra, "ws://") || !strings.HasSuffix(ra, "/echo") { + t.Errorf("Bad remote addr: %v", ra) } - var actual_msg = make([]byte, 512) - n, err := ws.Read(actual_msg) + la := conn.LocalAddr().String() + if !strings.HasPrefix(la, "http://") { + t.Errorf("Bad local addr: %v", la) + } + conn.Close() +} + +func TestCount(t *testing.T) { + once.Do(startServer) + + // websocket.Dial() + client, err := net.Dial("tcp", serverAddr) if err != nil { - t.Errorf("Read: error %v", err) + t.Fatal("dialing", err) } - actual_msg = actual_msg[0:n] - if !bytes.Equal(msg, actual_msg) { - t.Errorf("Echo: expected %q got %q", msg, actual_msg) + conn, err := NewClient(newConfig(t, "/count"), client) + if err != nil { + t.Errorf("WebSocket handshake error: %v", err) + return } - ws.Close() + + var count Count + count.S = "hello" + if err := JSON.Send(conn, count); err != nil { + t.Errorf("Write: %v", err) + } + if err := JSON.Receive(conn, &count); err != nil { + t.Errorf("Read: %v", err) + } + if count.N != 1 { + t.Errorf("count: expected %d got %d", 1, count.N) + } + if count.S != "hello" { + t.Errorf("count: expected %q got %q", "hello", count.S) + } + if err := JSON.Send(conn, count); err != nil { + t.Errorf("Write: %v", err) + } + if err := JSON.Receive(conn, &count); err != nil { + t.Errorf("Read: %v", err) + } + if count.N != 2 { + t.Errorf("count: expected %d got %d", 2, count.N) + } + if count.S != "hellohello" { + t.Errorf("count: expected %q got %q", "hellohello", count.S) + } + conn.Close() } func TestWithQuery(t *testing.T) { @@ -120,8 +163,13 @@ func TestWithQuery(t *testing.T) { t.Fatal("dialing", err) } - ws, err := newClient("/echo?q=v", "localhost", "http://localhost", - "ws://localhost/echo?q=v", "", client, handshake) + config := newConfig(t, "/echo") + config.Location, err = url.ParseRequest(fmt.Sprintf("ws://%s/echo?q=v", serverAddr)) + if err != nil { + t.Fatal("location url", err) + } + + ws, err := NewClient(config, client) if err != nil { t.Errorf("WebSocket handshake: %v", err) return @@ -137,8 +185,10 @@ func TestWithProtocol(t *testing.T) { t.Fatal("dialing", err) } - ws, err := newClient("/echo", "localhost", "http://localhost", - "ws://localhost/echo", "test", client, handshake) + config := newConfig(t, "/echo") + config.Protocol = append(config.Protocol, "test") + + ws, err := NewClient(config, client) if err != nil { t.Errorf("WebSocket handshake: %v", err) return @@ -167,29 +217,17 @@ func TestHTTP(t *testing.T) { } } -func TestHTTPDraft75(t *testing.T) { - once.Do(startServer) - - r, err := http.Get(fmt.Sprintf("http://%s/echoDraft75", serverAddr)) - if err != nil { - t.Errorf("Get: error %#v", err) - return - } - if r.StatusCode != http.StatusBadRequest { - t.Errorf("Get: got status %d", r.StatusCode) - } -} - func TestTrailingSpaces(t *testing.T) { // http://code.google.com/p/go/issues/detail?id=955 // The last runs of this create keys with trailing spaces that should not be // generated by the client. once.Do(startServer) + config := newConfig(t, "/echo") for i := 0; i < 30; i++ { // body - ws, err := Dial(fmt.Sprintf("ws://%s/echo", serverAddr), "", "http://localhost/") + ws, err := DialConfig(config) if err != nil { - t.Error("Dial failed:", err.String()) + t.Errorf("Dial #%d failed: %v", i, err) break } ws.Close() @@ -206,19 +244,18 @@ func TestSmallBuffer(t *testing.T) { if err != nil { t.Fatal("dialing", err) } - ws, err := newClient("/echo", "localhost", "http://localhost", - "ws://localhost/echo", "", client, handshake) + conn, err := NewClient(newConfig(t, "/echo"), client) if err != nil { t.Errorf("WebSocket handshake error: %v", err) return } msg := []byte("hello, world\n") - if _, err := ws.Write(msg); err != nil { + if _, err := conn.Write(msg); err != nil { t.Errorf("Write: %v", err) } var small_msg = make([]byte, 8) - n, err := ws.Read(small_msg) + n, err := conn.Read(small_msg) if err != nil { t.Errorf("Read: %v", err) } @@ -226,7 +263,7 @@ func TestSmallBuffer(t *testing.T) { t.Errorf("Echo: expected %q got %q", msg[:len(small_msg)], small_msg) } var second_msg = make([]byte, len(msg)) - n, err = ws.Read(second_msg) + n, err = conn.Read(second_msg) if err != nil { t.Errorf("Read: %v", err) } @@ -234,38 +271,5 @@ func TestSmallBuffer(t *testing.T) { if !bytes.Equal(msg[len(small_msg):], second_msg) { t.Errorf("Echo: expected %q got %q", msg[len(small_msg):], second_msg) } - ws.Close() - -} - -func testSkipLengthFrame(t *testing.T) { - b := []byte{'\x80', '\x01', 'x', 0, 'h', 'e', 'l', 'l', 'o', '\xff'} - buf := bytes.NewBuffer(b) - br := bufio.NewReader(buf) - bw := bufio.NewWriter(buf) - ws := newConn("http://127.0.0.1/", "ws://127.0.0.1/", "", bufio.NewReadWriter(br, bw), nil) - msg := make([]byte, 5) - n, err := ws.Read(msg) - if err != nil { - t.Errorf("Read: %v", err) - } - if !bytes.Equal(b[4:8], msg[0:n]) { - t.Errorf("Read: expected %q got %q", msg[4:8], msg[0:n]) - } -} - -func testSkipNoUTF8Frame(t *testing.T) { - b := []byte{'\x01', 'n', '\xff', 0, 'h', 'e', 'l', 'l', 'o', '\xff'} - buf := bytes.NewBuffer(b) - br := bufio.NewReader(buf) - bw := bufio.NewWriter(buf) - ws := newConn("http://127.0.0.1/", "ws://127.0.0.1/", "", bufio.NewReadWriter(br, bw), nil) - msg := make([]byte, 5) - n, err := ws.Read(msg) - if err != nil { - t.Errorf("Read: %v", err) - } - if !bytes.Equal(b[4:8], msg[0:n]) { - t.Errorf("Read: expected %q got %q", msg[4:8], msg[0:n]) - } + conn.Close() } diff --git a/libgo/go/xml/marshal.go b/libgo/go/xml/marshal.go index ea421c1..8396dba 100644 --- a/libgo/go/xml/marshal.go +++ b/libgo/go/xml/marshal.go @@ -39,10 +39,10 @@ type printer struct { // Marshal handles a pointer by marshalling the value it points at or, if the // pointer is nil, by writing nothing. Marshal handles an interface value by // marshalling the value it contains or, if the interface value is nil, by -// writing nothing. Marshal handles all other data by writing a single XML -// element containing the data. +// writing nothing. Marshal handles all other data by writing one or more XML +// elements containing the data. // -// The name of that XML element is taken from, in order of preference: +// The name for the XML elements is taken from, in order of preference: // - the tag on an XMLName field, if the data is a struct // - the value of an XMLName field of type xml.Name // - the tag of the struct field used to obtain the data @@ -58,6 +58,31 @@ type printer struct { // - a field with tag "innerxml" is written verbatim, // not subject to the usual marshalling procedure. // +// If a field uses a tag "a>b>c", then the element c will be nested inside +// parent elements a and b. Fields that appear next to each other that name +// the same parent will be enclosed in one XML element. For example: +// +// type Result struct { +// XMLName xml.Name `xml:"result"` +// FirstName string `xml:"person>name>first"` +// LastName string `xml:"person>name>last"` +// Age int `xml:"person>age"` +// } +// +// xml.Marshal(w, &Result{FirstName: "John", LastName: "Doe", Age: 42}) +// +// would be marshalled as: +// +// <result> +// <person> +// <name> +// <first>John</first> +// <last>Doe</last> +// </name> +// <age>42</age> +// </person> +// </result> +// // Marshal will return an error if asked to marshal a channel, function, or map. func Marshal(w io.Writer, v interface{}) (err os.Error) { p := &printer{bufio.NewWriter(w)} @@ -170,22 +195,25 @@ func (p *printer) marshalValue(val reflect.Value, name string) os.Error { bytes := val.Interface().([]byte) Escape(p, bytes) case reflect.Struct: + s := parentStack{printer: p} for i, n := 0, val.NumField(); i < n; i++ { if f := typ.Field(i); f.Name != "XMLName" && f.PkgPath == "" { name := f.Name + vf := val.Field(i) switch tag := f.Tag.Get("xml"); tag { case "": + s.trim(nil) case "chardata": if tk := f.Type.Kind(); tk == reflect.String { - Escape(p, []byte(val.Field(i).String())) + Escape(p, []byte(vf.String())) } else if tk == reflect.Slice { - if elem, ok := val.Field(i).Interface().([]byte); ok { + if elem, ok := vf.Interface().([]byte); ok { Escape(p, elem) } } continue case "innerxml": - iface := val.Field(i).Interface() + iface := vf.Interface() switch raw := iface.(type) { case []byte: p.Write(raw) @@ -197,14 +225,28 @@ func (p *printer) marshalValue(val reflect.Value, name string) os.Error { case "attr": continue default: - name = tag + parents := strings.Split(tag, ">") + if len(parents) == 1 { + parents, name = nil, tag + } else { + parents, name = parents[:len(parents)-1], parents[len(parents)-1] + if parents[0] == "" { + parents[0] = f.Name + } + } + + s.trim(parents) + if !(vf.Kind() == reflect.Ptr || vf.Kind() == reflect.Interface) || !vf.IsNil() { + s.push(parents[len(s.stack):]) + } } - if err := p.marshalValue(val.Field(i), name); err != nil { + if err := p.marshalValue(vf, name); err != nil { return err } } } + s.trim(nil) default: return &UnsupportedTypeError{typ} } @@ -217,6 +259,41 @@ func (p *printer) marshalValue(val reflect.Value, name string) os.Error { return nil } +type parentStack struct { + *printer + stack []string +} + +// trim updates the XML context to match the longest common prefix of the stack +// and the given parents. A closing tag will be written for every parent +// popped. Passing a zero slice or nil will close all the elements. +func (s *parentStack) trim(parents []string) { + split := 0 + for ; split < len(parents) && split < len(s.stack); split++ { + if parents[split] != s.stack[split] { + break + } + } + + for i := len(s.stack) - 1; i >= split; i-- { + s.WriteString("</") + s.WriteString(s.stack[i]) + s.WriteByte('>') + } + + s.stack = parents[:split] +} + +// push adds parent elements to the stack and writes open tags. +func (s *parentStack) push(parents []string) { + for i := 0; i < len(parents); i++ { + s.WriteString("<") + s.WriteString(parents[i]) + s.WriteByte('>') + } + s.stack = append(s.stack, parents...) +} + // A MarshalXMLError is returned when Marshal or MarshalIndent encounter a type // that cannot be converted into XML. type UnsupportedTypeError struct { diff --git a/libgo/go/xml/marshal_test.go b/libgo/go/xml/marshal_test.go index 5b972fa..ad3aa97 100644 --- a/libgo/go/xml/marshal_test.go +++ b/libgo/go/xml/marshal_test.go @@ -69,6 +69,41 @@ type SecretAgent struct { Obfuscate string `xml:"innerxml"` } +type NestedItems struct { + XMLName Name `xml:"result"` + Items []string `xml:">item"` + Item1 []string `xml:"Items>item1"` +} + +type NestedOrder struct { + XMLName Name `xml:"result"` + Field1 string `xml:"parent>c"` + Field2 string `xml:"parent>b"` + Field3 string `xml:"parent>a"` +} + +type MixedNested struct { + XMLName Name `xml:"result"` + A string `xml:"parent1>a"` + B string `xml:"b"` + C string `xml:"parent1>parent2>c"` + D string `xml:"parent1>d"` +} + +type NilTest struct { + A interface{} `xml:"parent1>parent2>a"` + B interface{} `xml:"parent1>b"` + C interface{} `xml:"parent1>parent2>c"` +} + +type Service struct { + XMLName Name `xml:"service"` + Domain *Domain `xml:"host>domain"` + Port *Port `xml:"host>port"` + Extra1 interface{} + Extra2 interface{} `xml:"host>extra2"` +} + var nilStruct *Ship var marshalTests = []struct { @@ -170,6 +205,94 @@ var marshalTests = []struct { `</passenger>` + `</spaceship>`, }, + // Test a>b + { + Value: NestedItems{Items: []string{}, Item1: []string{}}, + ExpectXML: `<result>` + + `<Items>` + + `</Items>` + + `</result>`, + }, + { + Value: NestedItems{Items: []string{}, Item1: []string{"A"}}, + ExpectXML: `<result>` + + `<Items>` + + `<item1>A</item1>` + + `</Items>` + + `</result>`, + }, + { + Value: NestedItems{Items: []string{"A", "B"}, Item1: []string{}}, + ExpectXML: `<result>` + + `<Items>` + + `<item>A</item>` + + `<item>B</item>` + + `</Items>` + + `</result>`, + }, + { + Value: NestedItems{Items: []string{"A", "B"}, Item1: []string{"C"}}, + ExpectXML: `<result>` + + `<Items>` + + `<item>A</item>` + + `<item>B</item>` + + `<item1>C</item1>` + + `</Items>` + + `</result>`, + }, + { + Value: NestedOrder{Field1: "C", Field2: "B", Field3: "A"}, + ExpectXML: `<result>` + + `<parent>` + + `<c>C</c>` + + `<b>B</b>` + + `<a>A</a>` + + `</parent>` + + `</result>`, + }, + { + Value: NilTest{A: "A", B: nil, C: "C"}, + ExpectXML: `<???>` + + `<parent1>` + + `<parent2><a>A</a></parent2>` + + `<parent2><c>C</c></parent2>` + + `</parent1>` + + `</???>`, + }, + { + Value: MixedNested{A: "A", B: "B", C: "C", D: "D"}, + ExpectXML: `<result>` + + `<parent1><a>A</a></parent1>` + + `<b>B</b>` + + `<parent1>` + + `<parent2><c>C</c></parent2>` + + `<d>D</d>` + + `</parent1>` + + `</result>`, + }, + { + Value: Service{Port: &Port{Number: "80"}}, + ExpectXML: `<service><host><port>80</port></host></service>`, + }, + { + Value: Service{}, + ExpectXML: `<service></service>`, + }, + { + Value: Service{Port: &Port{Number: "80"}, Extra1: "A", Extra2: "B"}, + ExpectXML: `<service>` + + `<host><port>80</port></host>` + + `<Extra1>A</Extra1>` + + `<host><extra2>B</extra2></host>` + + `</service>`, + }, + { + Value: Service{Port: &Port{Number: "80"}, Extra2: "example"}, + ExpectXML: `<service>` + + `<host><port>80</port></host>` + + `<host><extra2>example</extra2></host>` + + `</service>`, + }, } func TestMarshal(t *testing.T) { diff --git a/libgo/go/xml/read.go b/libgo/go/xml/read.go index 786b69f..f64e130 100644 --- a/libgo/go/xml/read.go +++ b/libgo/go/xml/read.go @@ -321,10 +321,9 @@ func (p *Parser) unmarshal(val reflect.Value, start *StartElement) os.Error { // Save v := sv.FieldByIndex(f.Index) - if _, ok := v.Interface().(Name); !ok { - return UnmarshalError(sv.Type().String() + " field XMLName does not have type xml.Name") + if _, ok := v.Interface().(Name); ok { + v.Set(reflect.ValueOf(start.Name)) } - v.Set(reflect.ValueOf(start.Name)) } // Assign attributes. diff --git a/libgo/go/xml/read_test.go b/libgo/go/xml/read_test.go index 2126da3..d39c2d5 100644 --- a/libgo/go/xml/read_test.go +++ b/libgo/go/xml/read_test.go @@ -369,3 +369,25 @@ var attrStruct = AttrTest{ Bool: true, }, } + +// test data for TestUnmarshalWithoutNameType + +const OK = "OK" +const withoutNameTypeData = ` +<?xml version="1.0" charset="utf-8"?> +<Test3 attr="OK" />` + +type TestThree struct { + XMLName bool `xml:"Test3"` // XMLName field without an xml.Name type + Attr string `xml:"attr"` +} + +func TestUnmarshalWithoutNameType(t *testing.T) { + var x TestThree + if err := Unmarshal(StringReader(withoutNameTypeData), &x); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if x.Attr != OK { + t.Fatalf("have %v\nwant %v", x.Attr, OK) + } +} diff --git a/libgo/go/xml/xml.go b/libgo/go/xml/xml.go index e7ba44e..85c24bc 100644 --- a/libgo/go/xml/xml.go +++ b/libgo/go/xml/xml.go @@ -390,7 +390,11 @@ func (p *Parser) popElement(t *EndElement) bool { // translations that were associated with the element we just closed. for p.stk != nil && p.stk.kind != stkStart { s := p.pop() - p.ns[s.name.Local] = s.name.Space, s.ok + if s.ok { + p.ns[s.name.Local] = s.name.Space + } else { + delete(p.ns, s.name.Local) + } } return true diff --git a/libgo/merge.sh b/libgo/merge.sh index 59e1df2..660a5f9 100755 --- a/libgo/merge.sh +++ b/libgo/merge.sh @@ -38,9 +38,9 @@ rm -rf ${OLDDIR} hg clone -r ${old_rev} ${repository} ${OLDDIR} rm -rf ${NEWDIR} -hg clone -u release ${repository} ${NEWDIR} +hg clone -u weekly ${repository} ${NEWDIR} -new_rev=`cd ${NEWDIR} && hg log -r release | sed 1q | sed -e 's/.*://'` +new_rev=`cd ${NEWDIR} && hg log -r weekly | sed 1q | sed -e 's/.*://'` merge() { name=$1 diff --git a/libgo/mksysinfo.sh b/libgo/mksysinfo.sh index 8838fd7..e493ef0 100755 --- a/libgo/mksysinfo.sh +++ b/libgo/mksysinfo.sh @@ -463,7 +463,7 @@ echo $msghdr | \ -e 's/msg_flags/Flags/' \ >> ${OUT} -# The ip_mreq struct +# The ip_mreq struct. grep '^type _ip_mreq ' gen-sysinfo.go | \ sed -e 's/_ip_mreq/IPMreq/' \ -e 's/imr_multiaddr/Multiaddr/' \ @@ -476,6 +476,14 @@ if grep 'type IPMreq ' ${OUT} > /dev/null 2>&1; then echo 'var SizeofIPMreq = int(unsafe.Sizeof(IPMreq{}))' >> ${OUT} fi +# The ipv6_mreq struct. +grep '^type _ipv6_mreq ' gen-sysinfo.go | \ + sed -e 's/_ipv6_mreq/IPv6Mreq/' \ + -e 's/ipv6mr_multiaddr/Multiaddr/' \ + -e 's/ipv6mr_interface/Interface/' \ + -e 's/_in6_addr/[16]byte/' \ + >> ${OUT} + # Try to guess the type to use for fd_set. fd_set=`grep '^type _fd_set ' gen-sysinfo.go || true` fds_bits_type="_C_long" @@ -504,6 +512,10 @@ grep '^type _passwd ' gen-sysinfo.go | \ grep '^const _TIOC' gen-sysinfo.go | \ sed -e 's/^\(const \)_\(TIOC[^= ]*\)\(.*\)$/\1\2 = _\2/' >> ${OUT} +# The ioctl flags for terminal control +grep '^const _TC[GS]ET' gen-sysinfo.go | \ + sed -e 's/^\(const \)_\(TC[GS]ET[^= ]*\)\(.*\)$/\1\2 = _\2/' >> ${OUT} + # ioctl constants. Might fall back to 0 if TIOCNXCL is missing, too, but # needs handling in syscalls.exec.go. if ! grep '^const _TIOCSCTTY ' gen-sysinfo.go >/dev/null 2>&1; then @@ -619,4 +631,33 @@ if grep 'type RtAttr ' ${OUT} > /dev/null 2>&1; then echo 'var SizeofRtAttr = int(unsafe.Sizeof(RtAttr{}))' >> ${OUT} fi +# The termios struct. +grep '^type _termios ' gen-sysinfo.go | \ + sed -e 's/_termios/Termios/' \ + -e 's/c_iflag/Iflag/' \ + -e 's/c_oflag/Oflag/' \ + -e 's/c_cflag/Cflag/' \ + -e 's/c_lflag/Lflag/' \ + -e 's/c_line/Line/' \ + -e 's/c_cc/Cc/' \ + -e 's/c_ispeed/Ispeed/' \ + -e 's/c_ospeed/Ospeed/' \ + >> ${OUT} + +# The termios constants. The ones starting with 'E' were picked up above. +for n in IGNBRK BRKINT IGNPAR PARMRK INPCK ISTRIP INLCR IGNCR ICRNL IUCLC \ + IXON IXANY IXOFF IMAXBEL IUTF8 OPOST OLCUC ONLCR OCRNL ONOCR ONLRET \ + OFILL OFDEL NLDLY NL0 NL1 CRDLY CR0 CR1 CR2 CR3 TABDLY BSDLY VTDLY \ + FFDLY CBAUD CBAUDEX CSIZE CSTOPB CREAD PARENB PARODD HUPCL CLOCAL \ + LOBLK CIBAUD CMSPAR CRTSCTS ISIG ICANON XCASE DEFECHK FLUSHO NOFLSH \ + TOSTOP PENDIN IEXTEN VINTR VQUIT VERASE VKILL VEOF VMIN VEOL VTIME VEOL2 \ + VSWTCH VSTART VSTOP VSUSP VDSUSP VLNEXT VWERASE VREPRINT VDISCARD VSTATUS \ + TCSANOW TCSADRAIN, TCSAFLUSH TCIFLUSH TCOFLUSH TCIOFLUSH TCOOFF TCOON \ + TCIOFF TCION B0 B50 B75 B110 B134 B150 B200 B300 B600 B1200 B1800 B2400 \ + B4800 B9600 B19200 B38400 B57600 B115200 B230400; do + + grep "^const _$n " gen-sysinfo.go | \ + sed -e 's/^\(const \)_\([^=]*\)\(.*\)$/\1\2 = _\2/' >> ${OUT} +done + exit $? diff --git a/libgo/runtime/arch.h b/libgo/runtime/arch.h new file mode 100644 index 0000000..0546a5d --- /dev/null +++ b/libgo/runtime/arch.h @@ -0,0 +1,8 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// FIXME: Ideally CacheLineSize would be dependent on the host architecture. +enum { + CacheLineSize = 64 +}; diff --git a/libgo/runtime/cpuprof.c b/libgo/runtime/cpuprof.c index 3797e1c..bec15ae 100644 --- a/libgo/runtime/cpuprof.c +++ b/libgo/runtime/cpuprof.c @@ -49,6 +49,7 @@ // in the situation when normally the goroutine "owns" handoff. #include "runtime.h" +#include "arch.h" #include "malloc.h" #include "array.h" diff --git a/libgo/runtime/go-append.c b/libgo/runtime/go-append.c index 261d85b..b1e882c 100644 --- a/libgo/runtime/go-append.c +++ b/libgo/runtime/go-append.c @@ -8,6 +8,7 @@ #include "go-panic.h" #include "array.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" /* We should be OK if we don't split the stack here, since the only diff --git a/libgo/runtime/go-byte-array-to-string.c b/libgo/runtime/go-byte-array-to-string.c index ab9e283..cfe1906 100644 --- a/libgo/runtime/go-byte-array-to-string.c +++ b/libgo/runtime/go-byte-array-to-string.c @@ -6,6 +6,7 @@ #include "go-string.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_string diff --git a/libgo/runtime/go-go.c b/libgo/runtime/go-go.c index 1391620..d56b8b1 100644 --- a/libgo/runtime/go-go.c +++ b/libgo/runtime/go-go.c @@ -17,6 +17,7 @@ #include "go-panic.h" #include "go-alloc.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" #ifdef USING_SPLIT_STACK @@ -561,7 +562,7 @@ __go_cachestats (void) /* Start the other threads after garbage collection. */ void -runtime_starttheworld (void) +runtime_starttheworld (bool extra __attribute__ ((unused))) { int i; pthread_t me; diff --git a/libgo/runtime/go-int-array-to-string.c b/libgo/runtime/go-int-array-to-string.c index ec07b87..1a37879 100644 --- a/libgo/runtime/go-int-array-to-string.c +++ b/libgo/runtime/go-int-array-to-string.c @@ -7,6 +7,7 @@ #include "go-assert.h" #include "go-string.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_string diff --git a/libgo/runtime/go-int-to-string.c b/libgo/runtime/go-int-to-string.c index af58015..e9645bf 100644 --- a/libgo/runtime/go-int-to-string.c +++ b/libgo/runtime/go-int-to-string.c @@ -6,6 +6,7 @@ #include "go-string.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_string diff --git a/libgo/runtime/go-main.c b/libgo/runtime/go-main.c index 37956d5..927a36c 100644 --- a/libgo/runtime/go-main.c +++ b/libgo/runtime/go-main.c @@ -19,6 +19,7 @@ #include "go-string.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" #undef int diff --git a/libgo/runtime/go-make-slice.c b/libgo/runtime/go-make-slice.c index d0e8369..a818a7f 100644 --- a/libgo/runtime/go-make-slice.c +++ b/libgo/runtime/go-make-slice.c @@ -12,6 +12,7 @@ #include "go-type.h" #include "array.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_open_array diff --git a/libgo/runtime/go-new.c b/libgo/runtime/go-new.c index 657978c..b1af5f2 100644 --- a/libgo/runtime/go-new.c +++ b/libgo/runtime/go-new.c @@ -6,6 +6,7 @@ #include "go-alloc.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" void * diff --git a/libgo/runtime/go-note.c b/libgo/runtime/go-note.c index 2b80b9b..62c229f 100644 --- a/libgo/runtime/go-note.c +++ b/libgo/runtime/go-note.c @@ -12,7 +12,7 @@ #include "runtime.h" /* We use a single global lock and condition variable. It would be - better to use a futex on Linux. */ + better to use a futex on GNU/Linux. */ static pthread_mutex_t note_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t note_cond = PTHREAD_COND_INITIALIZER; diff --git a/libgo/runtime/go-panic.c b/libgo/runtime/go-panic.c index c39ea9f..f3e182d 100644 --- a/libgo/runtime/go-panic.c +++ b/libgo/runtime/go-panic.c @@ -8,6 +8,7 @@ #include <stdlib.h> #include "runtime.h" +#include "arch.h" #include "malloc.h" #include "go-alloc.h" #include "go-defer.h" diff --git a/libgo/runtime/go-semacquire.c b/libgo/runtime/go-semacquire.c index 40fe2af..05b6377 100644 --- a/libgo/runtime/go-semacquire.c +++ b/libgo/runtime/go-semacquire.c @@ -13,9 +13,9 @@ /* We use a single global lock and condition variable. This is painful, since it will cause unnecessary contention, but is hard to - avoid in a portable manner. On Linux we can use futexes, but they - are unfortunately not exposed by libc and are thus also hard to use - portably. */ + avoid in a portable manner. On GNU/Linux we can use futexes, but + they are unfortunately not exposed by libc and are thus also hard + to use portably. */ static pthread_mutex_t sem_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t sem_cond = PTHREAD_COND_INITIALIZER; diff --git a/libgo/runtime/go-string-to-byte-array.c b/libgo/runtime/go-string-to-byte-array.c index 10c565e..8bae54b 100644 --- a/libgo/runtime/go-string-to-byte-array.c +++ b/libgo/runtime/go-string-to-byte-array.c @@ -7,6 +7,7 @@ #include "go-string.h" #include "array.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_open_array diff --git a/libgo/runtime/go-string-to-int-array.c b/libgo/runtime/go-string-to-int-array.c index f59df67..aff1468 100644 --- a/libgo/runtime/go-string-to-int-array.c +++ b/libgo/runtime/go-string-to-int-array.c @@ -8,6 +8,7 @@ #include "go-string.h" #include "array.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_open_array diff --git a/libgo/runtime/go-strplus.c b/libgo/runtime/go-strplus.c index e4dea9c..bfbe341 100644 --- a/libgo/runtime/go-strplus.c +++ b/libgo/runtime/go-strplus.c @@ -6,6 +6,7 @@ #include "go-string.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_string diff --git a/libgo/runtime/go-strslice.c b/libgo/runtime/go-strslice.c index 94ecee9..40ccac6 100644 --- a/libgo/runtime/go-strslice.c +++ b/libgo/runtime/go-strslice.c @@ -7,6 +7,7 @@ #include "go-string.h" #include "go-panic.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" struct __go_string diff --git a/libgo/runtime/goc2c.c b/libgo/runtime/goc2c.c index 32fbceb..fe413fe 100644 --- a/libgo/runtime/goc2c.c +++ b/libgo/runtime/goc2c.c @@ -219,13 +219,14 @@ getchar_skipping_comments(void) } /* - * Read and return a token. Tokens are delimited by whitespace or by - * [(),{}]. The latter are all returned as single characters. + * Read and return a token. Tokens are string or character literals + * or else delimited by whitespace or by [(),{}]. + * The latter are all returned as single characters. */ static char * read_token(void) { - int c; + int c, q; char *buf; unsigned int alc, off; const char* delims = "(),{}"; @@ -240,7 +241,26 @@ read_token(void) alc = 16; buf = xmalloc(alc + 1); off = 0; - if (strchr(delims, c) != NULL) { + if(c == '"' || c == '\'') { + q = c; + buf[off] = c; + ++off; + while (1) { + if (off+2 >= alc) { // room for c and maybe next char + alc *= 2; + buf = xrealloc(buf, alc + 1); + } + c = getchar_no_eof(); + buf[off] = c; + ++off; + if(c == q) + break; + if(c == '\\') { + buf[off] = getchar_no_eof(); + ++off; + } + } + } else if (strchr(delims, c) != NULL) { buf[off] = c; ++off; } else { diff --git a/libgo/runtime/malloc.goc b/libgo/runtime/malloc.goc index 2ea69ee..f8d4327 100644 --- a/libgo/runtime/malloc.goc +++ b/libgo/runtime/malloc.goc @@ -12,6 +12,7 @@ package runtime #include <stdlib.h> #include "go-alloc.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" #include "go-string.h" #include "interface.h" @@ -96,11 +97,12 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) m->mcache->next_sample -= size; else { // pick next profile time + // If you change this, also change allocmcache. if(rate > 0x3fffffff) // make 2*rate not overflow rate = 0x3fffffff; m->mcache->next_sample = runtime_fastrand1() % (2*rate); profile: - runtime_setblockspecial(v); + runtime_setblockspecial(v, true); runtime_MProf_Malloc(v, size); } } @@ -224,6 +226,7 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp) MCache* runtime_allocmcache(void) { + int32 rate; MCache *c; if(!__sync_bool_compare_and_swap(&m->mallocing, 0, 1)) @@ -239,6 +242,13 @@ runtime_allocmcache(void) mstats.mcache_sys = runtime_mheap.cachealloc.sys; runtime_unlock(&runtime_mheap); + // Set first allocation sample size. + rate = runtime_MemProfileRate; + if(rate > 0x3fffffff) // make 2*rate not overflow + rate = 0x3fffffff; + if(rate != 0) + c->next_sample = runtime_fastrand1() % (2*rate); + __sync_bool_compare_and_swap(&m->mallocing, 1, 0); if(__sync_bool_compare_and_swap(&m->gcing, 1, 0)) __go_run_goroutine_gc(2); @@ -280,6 +290,7 @@ runtime_mallocinit(void) byte *p; uintptr arena_size, bitmap_size; extern byte end[]; + byte *want; runtime_sizeof_C_MStats = sizeof(MStats); @@ -341,9 +352,13 @@ runtime_mallocinit(void) // not as an absolute requirement. If we ask for the end // of the data segment but the operating system requires // a little more space before we can start allocating, it will - // give out a slightly higher pointer. That's fine. - // Run with what we get back. - p = runtime_SysReserve(end, bitmap_size + arena_size); + // give out a slightly higher pointer. Except QEMU, which + // is buggy, as usual: it won't adjust the pointer upward. + // So adjust it upward a little bit ourselves: 1/4 MB to get + // away from the running binary image and then round up + // to a MB boundary. + want = (byte*)(((uintptr)end + (1<<18) + (1<<20) - 1)&~((1<<20)-1)); + p = runtime_SysReserve(want, bitmap_size + arena_size); if(p == nil) runtime_throw("runtime: cannot reserve arena virtual address space"); } @@ -418,8 +433,9 @@ runtime_mal(uintptr n) return runtime_mallocgc(n, 0, 1, 1); } -func new(n uint32) (ret *uint8) { - ret = runtime_mal(n); +func new(typ *Type) (ret *uint8) { + uint32 flag = typ->__code&GO_NO_POINTERS ? FlagNoPointers : 0; + ret = runtime_mallocgc(typ->__size, flag, 1, 1); } func Alloc(n uintptr) (p *byte) { @@ -444,9 +460,8 @@ func SetFinalizer(obj Eface, finalizer Eface) { const FuncType *ft; if(obj.__type_descriptor == nil) { - // runtime_printf("runtime.SetFinalizer: first argument is nil interface\n"); - throw: - runtime_throw("runtime.SetFinalizer"); + // runtime·printf("runtime.SetFinalizer: first argument is nil interface\n"); + goto throw; } if(obj.__type_descriptor->__code != GO_PTR) { // runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string); @@ -458,19 +473,21 @@ func SetFinalizer(obj Eface, finalizer Eface) { } ft = nil; if(finalizer.__type_descriptor != nil) { - if(finalizer.__type_descriptor->__code != GO_FUNC) { - badfunc: - // runtime_printf("runtime.SetFinalizer: second argument is %S, not func(%S)\n", *finalizer.type->string, *obj.type->string); - goto throw; - } + if(finalizer.__type_descriptor->__code != GO_FUNC) + goto badfunc; ft = (const FuncType*)finalizer.__type_descriptor; if(ft->__dotdotdot || ft->__in.__count != 1 || !__go_type_descriptors_equal(*(Type**)ft->__in.__values, obj.__type_descriptor)) goto badfunc; + } - if(runtime_getfinalizer(obj.__object, 0)) { - // runtime_printf("runtime.SetFinalizer: finalizer already set"); - goto throw; - } + if(!runtime_addfinalizer(obj.__object, finalizer.__type_descriptor != nil ? *(void**)finalizer.__object : nil, ft)) { + runtime_printf("runtime.SetFinalizer: finalizer already set\n"); + goto throw; } - runtime_addfinalizer(obj.__object, finalizer.__type_descriptor != nil ? *(void**)finalizer.__object : nil, ft); + return; + +badfunc: + // runtime_printf("runtime.SetFinalizer: second argument is %S, not func(%S)\n", *finalizer.type->string, *obj.type->string); +throw: + runtime_throw("runtime.SetFinalizer"); } diff --git a/libgo/runtime/malloc.h b/libgo/runtime/malloc.h index 3e813bb..1ccc2f0 100644 --- a/libgo/runtime/malloc.h +++ b/libgo/runtime/malloc.h @@ -120,6 +120,13 @@ enum #else MHeapMap_Bits = 20, #endif + + // Max number of threads to run garbage collection. + // 2, 3, and 4 are all plausible maximums depending + // on the hardware details of the machine. The second + // proc is the one that helps the most (after the first), + // so start with just 2 for now. + MaxGcproc = 2, }; // A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).) @@ -192,7 +199,7 @@ struct MStats uint64 nlookup; // number of pointer lookups uint64 nmalloc; // number of mallocs uint64 nfree; // number of frees - + // Statistics about malloc heap. // protected by mheap.Lock uint64 heap_alloc; // bytes allocated and still in use @@ -210,7 +217,7 @@ struct MStats uint64 mcache_inuse; // MCache structures uint64 mcache_sys; uint64 buckhash_sys; // profiling bucket hash table - + // Statistics about garbage collector. // Protected by stopping the world during GC. uint64 next_gc; // next GC (in heap_alloc time) @@ -219,7 +226,7 @@ struct MStats uint32 numgc; bool enablegc; bool debuggc; - + // Statistics about allocation size classes. struct { uint32 size; @@ -240,7 +247,7 @@ extern MStats mstats // // class_to_size[i] = largest size in class i // class_to_allocnpages[i] = number of pages to allocate when -// making new objects in class i +// making new objects in class i // class_to_transfercount[i] = number of objects to move when // taking a bunch of objects out of the central lists // and putting them in the thread free list. @@ -279,7 +286,7 @@ struct MCache int64 nmalloc; int64 nfree; } local_by_size[NumSizeClasses]; - + }; void* runtime_MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed); @@ -352,14 +359,14 @@ struct MHeap byte *arena_start; byte *arena_used; byte *arena_end; - + // central free lists for small size classes. // the union makes sure that the MCentrals are - // spaced 64 bytes apart, so that each MCentral.Lock + // spaced CacheLineSize bytes apart, so that each MCentral.Lock // gets its own cache line. union { MCentral; - byte pad[64]; + byte pad[CacheLineSize]; } central[NumSizeClasses]; FixAlloc spanalloc; // allocator for Span* @@ -387,7 +394,7 @@ int32 runtime_checking; void runtime_markspan(void *v, uintptr size, uintptr n, bool leftover); void runtime_unmarkspan(void *v, uintptr size); bool runtime_blockspecial(void*); -void runtime_setblockspecial(void*); +void runtime_setblockspecial(void*, bool); void runtime_purgecachedstats(M*); enum @@ -402,6 +409,8 @@ void runtime_Mprof_Init(void); void runtime_MProf_Malloc(void*, uintptr); void runtime_MProf_Free(void*, uintptr); void runtime_MProf_Mark(void (*scan)(byte *, int64)); +int32 runtime_helpgc(bool*); +void runtime_gchelper(void); // Malloc profiling settings. // Must match definition in extern.go. @@ -412,13 +421,6 @@ enum { }; extern int32 runtime_malloc_profile; -typedef struct Finalizer Finalizer; -struct Finalizer -{ - Finalizer *next; // for use by caller of getfinalizer - void (*fn)(void*); - void *arg; - const struct __go_func_type *ft; -}; - -Finalizer* runtime_getfinalizer(void*, bool); +struct __go_func_type; +bool runtime_getfinalizer(void *p, bool del, void (**fn)(void*), const struct __go_func_type **ft); +void runtime_walkfintab(void (*fn)(void*), void (*scan)(byte*, int64)); diff --git a/libgo/runtime/mcache.c b/libgo/runtime/mcache.c index 191b0d1..6c60aeb 100644 --- a/libgo/runtime/mcache.c +++ b/libgo/runtime/mcache.c @@ -7,6 +7,7 @@ // See malloc.h for an overview. #include "runtime.h" +#include "arch.h" #include "malloc.h" void* diff --git a/libgo/runtime/mcentral.c b/libgo/runtime/mcentral.c index cd3d6ca..b98a8d3 100644 --- a/libgo/runtime/mcentral.c +++ b/libgo/runtime/mcentral.c @@ -15,6 +15,7 @@ // so that it is faster to move those lists between MCaches and MCentrals. #include "runtime.h" +#include "arch.h" #include "malloc.h" static bool MCentral_Grow(MCentral *c); diff --git a/libgo/runtime/mem.c b/libgo/runtime/mem.c index 90c2c61..4267c55 100644 --- a/libgo/runtime/mem.c +++ b/libgo/runtime/mem.c @@ -2,6 +2,7 @@ #include <unistd.h> #include "runtime.h" +#include "arch.h" #include "malloc.h" #ifndef MAP_ANON diff --git a/libgo/runtime/mem_posix_memalign.c b/libgo/runtime/mem_posix_memalign.c index 2318be8..7d04f99 100644 --- a/libgo/runtime/mem_posix_memalign.c +++ b/libgo/runtime/mem_posix_memalign.c @@ -1,6 +1,7 @@ #include <errno.h> #include "runtime.h" +#include "arch.h" #include "malloc.h" void* diff --git a/libgo/runtime/mfinal.c b/libgo/runtime/mfinal.c index 04d58dd..db9a4fd 100644 --- a/libgo/runtime/mfinal.c +++ b/libgo/runtime/mfinal.c @@ -3,18 +3,17 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "arch.h" #include "malloc.h" -// Lock to protect finalizer data structures. -// Cannot reuse mheap.Lock because the finalizer -// maintenance requires allocation. -static Lock finlock; +enum { debug = 0 }; -void -runtime_initfintab() +typedef struct Fin Fin; +struct Fin { - runtime_initlock(&finlock); -} + void (*fn)(void*); + const struct __go_func_type *ft; +}; // Finalizer hash table. Direct hash, linear scan, at most 3/4 full. // Table size is power of 3 so that hash can be key % max. @@ -26,25 +25,43 @@ runtime_initfintab() typedef struct Fintab Fintab; struct Fintab { - void **key; - Finalizer **val; + Lock; + void **fkey; + Fin *val; int32 nkey; // number of non-nil entries in key int32 ndead; // number of dead (-1) entries in key int32 max; // size of key, val allocations }; +#define TABSZ 17 +#define TAB(p) (&fintab[((uintptr)(p)>>3)%TABSZ]) + +static struct { + Fintab; + uint8 pad[0 /* CacheLineSize - sizeof(Fintab) */]; +} fintab[TABSZ]; + +void +runtime_initfintab() +{ + int32 i; + + for(i=0; i<TABSZ; i++) + runtime_initlock(&fintab[i]); +} + static void -addfintab(Fintab *t, void *k, Finalizer *v) +addfintab(Fintab *t, void *k, void (*fn)(void*), const struct __go_func_type *ft) { int32 i, j; i = (uintptr)k % (uintptr)t->max; for(j=0; j<t->max; j++) { - if(t->key[i] == nil) { + if(t->fkey[i] == nil) { t->nkey++; goto ret; } - if(t->key[i] == (void*)-1) { + if(t->fkey[i] == (void*)-1) { t->ndead--; goto ret; } @@ -56,30 +73,32 @@ addfintab(Fintab *t, void *k, Finalizer *v) runtime_throw("finalizer table inconsistent"); ret: - t->key[i] = k; - t->val[i] = v; + t->fkey[i] = k; + t->val[i].fn = fn; + t->val[i].ft = ft; } -static Finalizer* -lookfintab(Fintab *t, void *k, bool del) +static bool +lookfintab(Fintab *t, void *k, bool del, Fin *f) { int32 i, j; - Finalizer *v; if(t->max == 0) - return nil; + return false; i = (uintptr)k % (uintptr)t->max; for(j=0; j<t->max; j++) { - if(t->key[i] == nil) - return nil; - if(t->key[i] == k) { - v = t->val[i]; + if(t->fkey[i] == nil) + return false; + if(t->fkey[i] == k) { + if(f) + *f = t->val[i]; if(del) { - t->key[i] = (void*)-1; - t->val[i] = nil; + t->fkey[i] = (void*)-1; + t->val[i].fn = nil; + t->val[i].ft = nil; t->ndead++; } - return v; + return true; } if(++i == t->max) i = 0; @@ -87,108 +106,123 @@ lookfintab(Fintab *t, void *k, bool del) // cannot happen - table is known to be non-full runtime_throw("finalizer table inconsistent"); - return nil; + return false; } -static Fintab fintab; - -// add finalizer; caller is responsible for making sure not already in table -void -runtime_addfinalizer(void *p, void (*f)(void*), const struct __go_func_type *ft) +static void +resizefintab(Fintab *tab) { Fintab newtab; + void *k; int32 i; - byte *base; - Finalizer *e; + + runtime_memclr((byte*)&newtab, sizeof newtab); + newtab.max = tab->max; + if(newtab.max == 0) + newtab.max = 3*3*3; + else if(tab->ndead < tab->nkey/2) { + // grow table if not many dead values. + // otherwise just rehash into table of same size. + newtab.max *= 3; + } + + newtab.fkey = runtime_mallocgc(newtab.max*sizeof newtab.fkey[0], FlagNoPointers, 0, 1); + newtab.val = runtime_mallocgc(newtab.max*sizeof newtab.val[0], 0, 0, 1); - e = nil; - if(f != nil) { - e = runtime_mal(sizeof *e); - e->fn = f; - e->ft = ft; + for(i=0; i<tab->max; i++) { + k = tab->fkey[i]; + if(k != nil && k != (void*)-1) + addfintab(&newtab, k, tab->val[i].fn, tab->val[i].ft); } + + runtime_free(tab->fkey); + runtime_free(tab->val); + + tab->fkey = newtab.fkey; + tab->val = newtab.val; + tab->nkey = newtab.nkey; + tab->ndead = newtab.ndead; + tab->max = newtab.max; +} +bool +runtime_addfinalizer(void *p, void (*f)(void*), const struct __go_func_type *ft) +{ + Fintab *tab; + byte *base; + bool ret = false; + + if(debug) { + if(!runtime_mlookup(p, &base, nil, nil) || p != base) + runtime_throw("addfinalizer on invalid pointer"); + } + if(!__sync_bool_compare_and_swap(&m->holds_finlock, 0, 1)) runtime_throw("finalizer deadlock"); - runtime_lock(&finlock); - if(!runtime_mlookup(p, &base, nil, nil) || p != base) { - runtime_unlock(&finlock); - __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); - runtime_throw("addfinalizer on invalid pointer"); - } + tab = TAB(p); + runtime_lock(tab); if(f == nil) { - lookfintab(&fintab, p, 1); + if(lookfintab(tab, p, true, nil)) + runtime_setblockspecial(p, false); + ret = true; goto unlock; } - if(lookfintab(&fintab, p, 0)) { - runtime_unlock(&finlock); - __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); - runtime_throw("double finalizer"); + if(lookfintab(tab, p, false, nil)) { + ret = false; + goto unlock; } - runtime_setblockspecial(p); - if(fintab.nkey >= fintab.max/2+fintab.max/4) { + if(tab->nkey >= tab->max/2+tab->max/4) { // keep table at most 3/4 full: // allocate new table and rehash. - - runtime_memclr((byte*)&newtab, sizeof newtab); - newtab.max = fintab.max; - if(newtab.max == 0) - newtab.max = 3*3*3; - else if(fintab.ndead < fintab.nkey/2) { - // grow table if not many dead values. - // otherwise just rehash into table of same size. - newtab.max *= 3; - } - - newtab.key = runtime_mallocgc(newtab.max*sizeof newtab.key[0], FlagNoPointers, 0, 1); - newtab.val = runtime_mallocgc(newtab.max*sizeof newtab.val[0], 0, 0, 1); - - for(i=0; i<fintab.max; i++) { - void *k; - - k = fintab.key[i]; - if(k != nil && k != (void*)-1) - addfintab(&newtab, k, fintab.val[i]); - } - runtime_free(fintab.key); - runtime_free(fintab.val); - fintab = newtab; + resizefintab(tab); } - addfintab(&fintab, p, e); + addfintab(tab, p, f, ft); + runtime_setblockspecial(p, true); + ret = true; + unlock: - runtime_unlock(&finlock); + runtime_unlock(tab); __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); if(__sync_bool_compare_and_swap(&m->gcing_for_finlock, 1, 0)) { __go_run_goroutine_gc(200); } + + return ret; } // get finalizer; if del, delete finalizer. -// caller is responsible for updating RefHasFinalizer bit. -Finalizer* -runtime_getfinalizer(void *p, bool del) +// caller is responsible for updating RefHasFinalizer (special) bit. +bool +runtime_getfinalizer(void *p, bool del, void (**fn)(void*), const struct __go_func_type **ft) { - Finalizer *f; + Fintab *tab; + bool res; + Fin f; if(!__sync_bool_compare_and_swap(&m->holds_finlock, 0, 1)) runtime_throw("finalizer deadlock"); - runtime_lock(&finlock); - f = lookfintab(&fintab, p, del); - runtime_unlock(&finlock); + tab = TAB(p); + runtime_lock(tab); + res = lookfintab(tab, p, del, &f); + runtime_unlock(tab); __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); if(__sync_bool_compare_and_swap(&m->gcing_for_finlock, 1, 0)) { __go_run_goroutine_gc(201); } - return f; + if(res==false) + return false; + *fn = f.fn; + *ft = f.ft; + return true; } void @@ -196,18 +230,22 @@ runtime_walkfintab(void (*fn)(void*), void (*scan)(byte *, int64)) { void **key; void **ekey; + int32 i; if(!__sync_bool_compare_and_swap(&m->holds_finlock, 0, 1)) runtime_throw("finalizer deadlock"); - scan((byte*)&fintab, sizeof fintab); - runtime_lock(&finlock); - key = fintab.key; - ekey = key + fintab.max; - for(; key < ekey; key++) - if(*key != nil && *key != ((void*)-1)) - fn(*key); - runtime_unlock(&finlock); + for(i=0; i<TABSZ; i++) { + runtime_lock(&fintab[i]); + key = fintab[i].fkey; + ekey = key + fintab[i].max; + for(; key < ekey; key++) + if(*key != nil && *key != ((void*)-1)) + fn(*key); + scan((byte*)&fintab[i].fkey, sizeof(void*)); + scan((byte*)&fintab[i].val, sizeof(void*)); + runtime_unlock(&fintab[i]); + } __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); if(__sync_bool_compare_and_swap(&m->gcing_for_finlock, 1, 0)) { diff --git a/libgo/runtime/mfixalloc.c b/libgo/runtime/mfixalloc.c index c05583d..109cfe8 100644 --- a/libgo/runtime/mfixalloc.c +++ b/libgo/runtime/mfixalloc.c @@ -7,6 +7,7 @@ // See malloc.h for overview. #include "runtime.h" +#include "arch.h" #include "malloc.h" // Initialize f to allocate objects of the given size, diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index 900ebde..cb58525 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -5,13 +5,14 @@ // Garbage collector. #include "runtime.h" +#include "arch.h" #include "malloc.h" enum { Debug = 0, - UseCas = 1, PtrSize = sizeof(void*), - + DebugMark = 0, // run second pass to check mark + // Four bits per word (see #defines below). wordsPerBitmapWord = sizeof(void*)*8/4, bitShift = sizeof(void*)*8/4, @@ -50,28 +51,72 @@ enum { #define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial) +// TODO: Make these per-M. static uint64 nlookup; static uint64 nsizelookup; static uint64 naddrlookup; +static uint64 nhandoff; + static int32 gctrace; typedef struct Workbuf Workbuf; struct Workbuf { Workbuf *next; - uintptr nw; - byte *w[2048-2]; + uintptr nobj; + byte *obj[512-2]; +}; + +typedef struct Finalizer Finalizer; +struct Finalizer +{ + void (*fn)(void*); + void *arg; + const struct __go_func_type *ft; +}; + +typedef struct FinBlock FinBlock; +struct FinBlock +{ + FinBlock *alllink; + FinBlock *next; + int32 cnt; + int32 cap; + Finalizer fin[1]; }; static bool finstarted; static pthread_mutex_t finqlock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t finqcond = PTHREAD_COND_INITIALIZER; -static Finalizer *finq; +static FinBlock *finq; // list of finalizers that are to be executed +static FinBlock *finc; // cache of free blocks +static FinBlock *allfin; // list of all blocks +static Lock finlock; static int32 fingwait; static void runfinq(void*); static Workbuf* getempty(Workbuf*); static Workbuf* getfull(Workbuf*); +static void putempty(Workbuf*); +static Workbuf* handoff(Workbuf*); + +static struct { + Lock fmu; + Workbuf *full; + Lock emu; + Workbuf *empty; + uint32 nproc; + volatile uint32 nwait; + volatile uint32 ndone; + Note alldone; + Lock markgate; + Lock sweepgate; + MSpan *spans; + + Lock; + byte *chunk; + uintptr nchunk; +} work; // scanblock scans a block of n bytes starting at pointer b for references // to other objects, scanning any it finds recursively until there are no @@ -82,13 +127,14 @@ static Workbuf* getfull(Workbuf*); static void scanblock(byte *b, int64 n) { - byte *obj, *arena_start, *p; + byte *obj, *arena_start, *arena_used, *p; void **vp; - uintptr size, *bitp, bits, shift, i, j, x, xbits, off; + uintptr size, *bitp, bits, shift, i, j, x, xbits, off, nobj, nproc; MSpan *s; PageID k; - void **bw, **w, **ew; + void **wp; Workbuf *wbuf; + bool keepworking; if((int64)(uintptr)n != n || n < 0) { // runtime_printf("scanblock %p %lld\n", b, (long long)n); @@ -97,11 +143,19 @@ scanblock(byte *b, int64 n) // Memory arena parameters. arena_start = runtime_mheap.arena_start; - + arena_used = runtime_mheap.arena_used; + nproc = work.nproc; + wbuf = nil; // current work buffer - ew = nil; // end of work buffer - bw = nil; // beginning of work buffer - w = nil; // current pointer into work buffer + wp = nil; // storage for next queued pointer (write pointer) + nobj = 0; // number of queued objects + + // Scanblock helpers pass b==nil. + // The main proc needs to return to make more + // calls to scanblock. But if work.nproc==1 then + // might as well process blocks as soon as we + // have them. + keepworking = b == nil || work.nproc == 1; // Align b to a word boundary. off = (uintptr)b & (PtrSize-1); @@ -117,17 +171,17 @@ scanblock(byte *b, int64 n) runtime_printf("scanblock %p %lld\n", b, (long long) n); vp = (void**)b; - n /= PtrSize; + n >>= (2+PtrSize/8); /* n /= PtrSize (4 or 8) */ for(i=0; i<(uintptr)n; i++) { obj = (byte*)vp[i]; - + // Words outside the arena cannot be pointers. - if((byte*)obj < arena_start || (byte*)obj >= runtime_mheap.arena_used) + if((byte*)obj < arena_start || (byte*)obj >= arena_used) continue; - + // obj may be a pointer to a live object. // Try to find the beginning of the object. - + // Round down to word boundary. obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); @@ -185,47 +239,72 @@ scanblock(byte *b, int64 n) found: // Now we have bits, bitp, and shift correct for // obj pointing at the base of the object. - // If not allocated or already marked, done. - if((bits & bitAllocated) == 0 || (bits & bitMarked) != 0) + // Only care about allocated and not marked. + if((bits & (bitAllocated|bitMarked)) != bitAllocated) continue; - *bitp |= bitMarked<<shift; + if(nproc == 1) + *bitp |= bitMarked<<shift; + else { + for(;;) { + x = *bitp; + if(x & (bitMarked<<shift)) + goto continue_obj; + if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift)))) + break; + } + } // If object has no pointers, don't need to scan further. if((bits & bitNoPointers) != 0) continue; + // If another proc wants a pointer, give it some. + if(nobj > 4 && work.nwait > 0 && work.full == nil) { + wbuf->nobj = nobj; + wbuf = handoff(wbuf); + nobj = wbuf->nobj; + wp = (void**)(wbuf->obj + nobj); + } + // If buffer is full, get a new one. - if(w >= ew) { + if(wbuf == nil || nobj >= nelem(wbuf->obj)) { + if(wbuf != nil) + wbuf->nobj = nobj; wbuf = getempty(wbuf); - bw = (void**)wbuf->w; - w = bw; - ew = bw + nelem(wbuf->w); + wp = (void**)(wbuf->obj); + nobj = 0; } - *w++ = obj; + *wp++ = obj; + nobj++; + continue_obj:; } - + // Done scanning [b, b+n). Prepare for the next iteration of // the loop by setting b and n to the parameters for the next block. - // Fetch b from the work buffers. - if(w <= bw) { + // Fetch b from the work buffer. + if(nobj == 0) { + if(!keepworking) { + putempty(wbuf); + return; + } // Emptied our buffer: refill. wbuf = getfull(wbuf); if(wbuf == nil) - break; - bw = (void**)wbuf->w; - ew = (void**)(wbuf->w + nelem(wbuf->w)); - w = bw+wbuf->nw; + return; + nobj = wbuf->nobj; + wp = (void**)(wbuf->obj + wbuf->nobj); } - b = *--w; - + b = *--wp; + nobj--; + // Figure out n = size of b. Start by loading bits for b. off = (uintptr*)b - (uintptr*)arena_start; bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; xbits = *bitp; bits = xbits >> shift; - + // Might be small; look for nearby block boundary. // A block boundary is marked by either bitBlockBoundary // or bitAllocated being set (see notes near their definition). @@ -244,12 +323,12 @@ scanblock(byte *b, int64 n) // apply a mask to keep only the bits corresponding // to shift+j < bitShift aka j < bitShift-shift. bits &= (boundary<<(bitShift-shift)) - boundary; - + // A block boundary j words before b is indicated by // xbits>>(shift-j) & boundary // (assuming shift >= j). There is no cleverness here // avoid the test, because when j gets too large the shift - // turns negative, which is undefined in C. + // turns negative, which is undefined in C. for(j=1; j<bitShift; j++) { if(((bits>>j)&boundary) != 0 || (shift>=j && ((xbits>>(shift-j))&boundary) != 0)) { @@ -257,7 +336,7 @@ scanblock(byte *b, int64 n) goto scan; } } - + // Fall back to asking span about size class. // (Manually inlined copy of MHeap_Lookup.) nlookup++; @@ -274,29 +353,123 @@ scanblock(byte *b, int64 n) } } -static struct { - Workbuf *full; - Workbuf *empty; - byte *chunk; - uintptr nchunk; -} work; +// debug_scanblock is the debug copy of scanblock. +// it is simpler, slower, single-threaded, recursive, +// and uses bitSpecial as the mark bit. +static void +debug_scanblock(byte *b, int64 n) +{ + byte *obj, *p; + void **vp; + uintptr size, *bitp, bits, shift, i, xbits, off; + MSpan *s; + + if(!DebugMark) + runtime_throw("debug_scanblock without DebugMark"); + + if((int64)(uintptr)n != n || n < 0) { + //runtime_printf("debug_scanblock %p %D\n", b, n); + runtime_throw("debug_scanblock"); + } + + // Align b to a word boundary. + off = (uintptr)b & (PtrSize-1); + if(off != 0) { + b += PtrSize - off; + n -= PtrSize - off; + } + + vp = (void**)b; + n /= PtrSize; + for(i=0; i<(uintptr)n; i++) { + obj = (byte*)vp[i]; + + // Words outside the arena cannot be pointers. + if((byte*)obj < runtime_mheap.arena_start || (byte*)obj >= runtime_mheap.arena_used) + continue; + + // Round down to word boundary. + obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); + + // Consult span table to find beginning. + s = runtime_MHeap_LookupMaybe(&runtime_mheap, obj); + if(s == nil) + continue; + + + p = (byte*)((uintptr)s->start<<PageShift); + if(s->sizeclass == 0) { + obj = p; + size = (uintptr)s->npages<<PageShift; + } else { + if((byte*)obj >= (byte*)s->limit) + continue; + size = runtime_class_to_size[s->sizeclass]; + int32 i = ((byte*)obj - p)/size; + obj = p+i*size; + } + + // Now that we know the object header, reload bits. + off = (uintptr*)obj - (uintptr*)runtime_mheap.arena_start; + bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + xbits = *bitp; + bits = xbits >> shift; + + // Now we have bits, bitp, and shift correct for + // obj pointing at the base of the object. + // If not allocated or already marked, done. + if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // NOTE: bitSpecial not bitMarked + continue; + *bitp |= bitSpecial<<shift; + if(!(bits & bitMarked)) + runtime_printf("found unmarked block %p in %p\n", obj, vp+i); + + // If object has no pointers, don't need to scan further. + if((bits & bitNoPointers) != 0) + continue; + + debug_scanblock(obj, size); + } +} // Get an empty work buffer off the work.empty list, // allocating new buffers as needed. static Workbuf* getempty(Workbuf *b) { - if(b != nil) { - b->nw = nelem(b->w); - b->next = work.full; - work.full = b; - } - b = work.empty; - if(b != nil) { - work.empty = b->next; - return b; + if(work.nproc == 1) { + // Put b on full list. + if(b != nil) { + b->next = work.full; + work.full = b; + } + // Grab from empty list if possible. + b = work.empty; + if(b != nil) { + work.empty = b->next; + goto haveb; + } + } else { + // Put b on full list. + if(b != nil) { + runtime_lock(&work.fmu); + b->next = work.full; + work.full = b; + runtime_unlock(&work.fmu); + } + // Grab from empty list if possible. + runtime_lock(&work.emu); + b = work.empty; + if(b != nil) + work.empty = b->next; + runtime_unlock(&work.emu); + if(b != nil) + goto haveb; } - + + // Need to allocate. + runtime_lock(&work); if(work.nchunk < sizeof *b) { work.nchunk = 1<<20; work.chunk = runtime_SysAlloc(work.nchunk); @@ -304,25 +477,121 @@ getempty(Workbuf *b) b = (Workbuf*)work.chunk; work.chunk += sizeof *b; work.nchunk -= sizeof *b; + runtime_unlock(&work); + +haveb: + b->nobj = 0; return b; } +static void +putempty(Workbuf *b) +{ + if(b == nil) + return; + + if(work.nproc == 1) { + b->next = work.empty; + work.empty = b; + return; + } + + runtime_lock(&work.emu); + b->next = work.empty; + work.empty = b; + runtime_unlock(&work.emu); +} + // Get a full work buffer off the work.full list, or return nil. static Workbuf* getfull(Workbuf *b) { - if(b != nil) { - b->nw = 0; - b->next = work.empty; - work.empty = b; + int32 i; + Workbuf *b1; + + if(work.nproc == 1) { + // Put b on empty list. + if(b != nil) { + b->next = work.empty; + work.empty = b; + } + // Grab from full list if possible. + // Since work.nproc==1, no one else is + // going to give us work. + b = work.full; + if(b != nil) + work.full = b->next; + return b; + } + + putempty(b); + + // Grab buffer from full list if possible. + for(;;) { + b1 = work.full; + if(b1 == nil) + break; + runtime_lock(&work.fmu); + if(work.full != nil) { + b1 = work.full; + work.full = b1->next; + runtime_unlock(&work.fmu); + return b1; + } + runtime_unlock(&work.fmu); + } + + runtime_xadd(&work.nwait, +1); + for(i=0;; i++) { + b1 = work.full; + if(b1 != nil) { + runtime_lock(&work.fmu); + if(work.full != nil) { + runtime_xadd(&work.nwait, -1); + b1 = work.full; + work.full = b1->next; + runtime_unlock(&work.fmu); + return b1; + } + runtime_unlock(&work.fmu); + continue; + } + if(work.nwait == work.nproc) + return nil; + if(i < 10) + runtime_procyield(20); + else if(i < 20) + runtime_osyield(); + else + runtime_usleep(100); } - b = work.full; - if(b != nil) - work.full = b->next; - return b; } -// Scanstack calls scanblock on each of gp's stack segments. +static Workbuf* +handoff(Workbuf *b) +{ + int32 n; + Workbuf *b1; + + // Make new buffer with half of b's pointers. + b1 = getempty(nil); + n = b->nobj/2; + b->nobj -= n; + b1->nobj = n; + runtime_memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]); + nhandoff += n; + + // Put b on full list - let first half of b get stolen. + runtime_lock(&work.fmu); + b->next = work.full; + work.full = b; + runtime_unlock(&work.fmu); + + return b1; +} + +// Markfin calls scanblock on the blocks that have finalizers: +// the things pointed at cannot be freed until the finalizers have run. static void markfin(void *v) { @@ -355,11 +624,22 @@ __go_register_gc_roots (struct root_list* r) roots = r; } -// Mark static void -mark(void) +debug_markfin(void *v) +{ + uintptr size; + + if(!runtime_mlookup(v, (byte**)&v, &size, nil)) + runtime_throw("debug_mark - finalizer inconsistency"); + debug_scanblock(v, size); +} + +// Mark +static void +mark(void (*scan)(byte*, int64)) { struct root_list *pl; + FinBlock *fb; for(pl = roots; pl != nil; pl = pl->next) { struct root* pr = &pl->roots[0]; @@ -372,18 +652,63 @@ mark(void) } } - scanblock((byte*)&m0, sizeof m0); - scanblock((byte*)&finq, sizeof finq); - runtime_MProf_Mark(scanblock); + scan((byte*)&m0, sizeof m0); + scan((byte*)&finq, sizeof finq); + runtime_MProf_Mark(scan); // mark stacks - __go_scanstacks(scanblock); + __go_scanstacks(scan); // mark things pointed at by objects with finalizers - runtime_walkfintab(markfin, scanblock); + if(scan == debug_scanblock) + runtime_walkfintab(debug_markfin, scan); + else + runtime_walkfintab(markfin, scan); + + for(fb=allfin; fb; fb=fb->alllink) + scanblock((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0])); + + // in multiproc mode, join in the queued work. + scan(nil, 0); } -// Sweep frees or calls finalizers for blocks not marked in the mark phase. +static bool +handlespecial(byte *p, uintptr size) +{ + void (*fn)(void*); + const struct __go_func_type *ft; + FinBlock *block; + Finalizer *f; + + if(!runtime_getfinalizer(p, true, &fn, &ft)) { + runtime_setblockspecial(p, false); + runtime_MProf_Free(p, size); + return false; + } + + runtime_lock(&finlock); + if(finq == nil || finq->cnt == finq->cap) { + if(finc == nil) { + finc = runtime_SysAlloc(PageSize); + finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1; + finc->alllink = allfin; + allfin = finc; + } + block = finc; + finc = block->next; + block->next = finq; + finq = block; + } + f = &finq->fin[finq->cnt]; + finq->cnt++; + f->fn = fn; + f->ft = ft; + f->arg = p; + runtime_unlock(&finlock); + return true; +} + +// Sweep frees or collects finalizers for blocks not marked in the mark phase. // It clears the mark bits in preparation for the next GC round. static void sweep(void) @@ -393,9 +718,17 @@ sweep(void) uintptr size; byte *p; MCache *c; - Finalizer *f; + byte *arena_start; + + arena_start = runtime_mheap.arena_start; + + for(;;) { + s = work.spans; + if(s == nil) + break; + if(!runtime_casp(&work.spans, s, s->allnext)) + continue; - for(s = runtime_mheap.allspans; s != nil; s = s->allnext) { if(s->state != MSpanInUse) continue; @@ -410,13 +743,15 @@ sweep(void) npages = runtime_class_to_allocnpages[cl]; n = (npages << PageShift) / size; } - - // sweep through n objects of given size starting at p. + + // Sweep through n objects of given size starting at p. + // This thread owns the span now, so it can manipulate + // the block bitmap without atomic operations. for(; n > 0; n--, p += size) { uintptr off, *bitp, shift, bits; - off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start; - bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)p - (uintptr*)arena_start; + bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; bits = *bitp>>shift; @@ -424,20 +759,21 @@ sweep(void) continue; if((bits & bitMarked) != 0) { + if(DebugMark) { + if(!(bits & bitSpecial)) + runtime_printf("found spurious mark on %p\n", p); + *bitp &= ~(bitSpecial<<shift); + } *bitp &= ~(bitMarked<<shift); continue; } - if((bits & bitSpecial) != 0) { - // Special means it has a finalizer or is being profiled. - f = runtime_getfinalizer(p, 1); - if(f != nil) { - f->arg = p; - f->next = finq; - finq = f; + // Special means it has a finalizer or is being profiled. + // In DebugMark mode, the bit has been coopted so + // we have to assume all blocks are special. + if(DebugMark || (bits & bitSpecial) != 0) { + if(handlespecial(p, size)) continue; - } - runtime_MProf_Free(p, size); } // Mark freed; restore block boundary bit. @@ -464,6 +800,23 @@ sweep(void) static pthread_mutex_t gcsema = PTHREAD_MUTEX_INITIALIZER; +void +runtime_gchelper(void) +{ + // Wait until main proc is ready for mark help. + runtime_lock(&work.markgate); + runtime_unlock(&work.markgate); + scanblock(nil, 0); + + // Wait until main proc is ready for sweep help. + runtime_lock(&work.sweepgate); + runtime_unlock(&work.sweepgate); + sweep(); + + if(runtime_xadd(&work.ndone, +1) == work.nproc-1) + runtime_notewakeup(&work.alldone); +} + // Initialized from $GOGC. GOGC=off means no gc. // // Next gc is after we've allocated an extra amount of @@ -481,7 +834,7 @@ runtime_gc(int32 force __attribute__ ((unused))) int64 t0, t1, t2, t3; uint64 heap0, heap1, obj0, obj1; char *p; - Finalizer *fp; + bool extra; // The gc is turned off (via enablegc) until // the bootstrap has completed. @@ -502,10 +855,16 @@ runtime_gc(int32 force __attribute__ ((unused))) gcpercent = -1; else gcpercent = runtime_atoi(p); - + p = runtime_getenv("GOGCTRACE"); if(p != nil) gctrace = runtime_atoi(p); + + runtime_initlock(&work.fmu); + runtime_initlock(&work.emu); + runtime_initlock(&work.markgate); + runtime_initlock(&work.sweepgate); + runtime_initlock(&work.Lock); } if(gcpercent < 0) return; @@ -522,20 +881,42 @@ runtime_gc(int32 force __attribute__ ((unused))) nlookup = 0; nsizelookup = 0; naddrlookup = 0; + nhandoff = 0; m->gcing = 1; runtime_stoptheworld(); - if(runtime_mheap.Lock.key != 0) - runtime_throw("runtime_mheap locked during gc"); __go_cachestats(); heap0 = mstats.heap_alloc; obj0 = mstats.nmalloc - mstats.nfree; - mark(); + runtime_lock(&work.markgate); + runtime_lock(&work.sweepgate); + + extra = false; + work.nproc = 1; +#if 0 + if(runtime_gomaxprocs > 1 && runtime_ncpu > 1) { + runtime_noteclear(&work.alldone); + work.nproc += runtime_helpgc(&extra); + } +#endif + work.nwait = 0; + work.ndone = 0; + + runtime_unlock(&work.markgate); // let the helpers in + mark(scanblock); + if(DebugMark) + mark(debug_scanblock); t1 = runtime_nanotime(); + + work.spans = runtime_mheap.allspans; + runtime_unlock(&work.sweepgate); // let the helpers in sweep(); + if(work.nproc > 1) + runtime_notesleep(&work.alldone); t2 = runtime_nanotime(); + __go_stealcache(); __go_cachestats(); @@ -553,21 +934,28 @@ runtime_gc(int32 force __attribute__ ((unused))) mstats.numgc++; if(mstats.debuggc) runtime_printf("pause %llu\n", (unsigned long long)t3-t0); - + if(gctrace) { - runtime_printf("gc%d: %llu+%llu+%llu ms %llu -> %llu MB %llu -> %llu (%llu-%llu) objects %llu pointer lookups (%llu size, %llu addr)\n", + runtime_printf("gc%d: %llu+%llu+%llu ms %llu -> %llu MB %llu -> %llu (%llu-%llu) objects %llu pointer lookups (%llu size, %llu addr) %llu handoff\n", mstats.numgc, (unsigned long long)(t1-t0)/1000000, (unsigned long long)(t2-t1)/1000000, (unsigned long long)(t3-t2)/1000000, (unsigned long long)heap0>>20, (unsigned long long)heap1>>20, (unsigned long long)obj0, (unsigned long long)obj1, (unsigned long long)mstats.nmalloc, (unsigned long long)mstats.nfree, - (unsigned long long)nlookup, (unsigned long long)nsizelookup, (unsigned long long)naddrlookup); + (unsigned long long)nlookup, (unsigned long long)nsizelookup, (unsigned long long)naddrlookup, (unsigned long long) nhandoff); } pthread_mutex_unlock(&gcsema); - runtime_starttheworld(); + + // If we could have used another helper proc, start one now, + // in the hope that it will be available next time. + // It would have been even better to start it before the collection, + // but doing so requires allocating memory, so it's tricky to + // coordinate. This lazy approach works out in practice: + // we don't mind if the first couple gc rounds don't have quite + // the maximum number of procs. + runtime_starttheworld(extra); // finqlock is still held. - fp = finq; - if(fp != nil) { + if(finq != nil) { // kick off or wake up goroutine to run queued finalizers if(!finstarted) { __go_go(runfinq, nil); @@ -601,37 +989,44 @@ runtime_UpdateMemStats(void) __go_cachestats(); m->gcing = 0; pthread_mutex_unlock(&gcsema); - runtime_starttheworld(); + runtime_starttheworld(false); } static void runfinq(void* dummy) { - Finalizer *f, *next; + Finalizer *f; + FinBlock *fb, *next; + uint32 i; USED(dummy); for(;;) { pthread_mutex_lock(&finqlock); - f = finq; + fb = finq; finq = nil; - if(f == nil) { + if(fb == nil) { fingwait = 1; pthread_cond_wait(&finqcond, &finqlock); pthread_mutex_unlock(&finqlock); continue; } pthread_mutex_unlock(&finqlock); - for(; f; f=next) { - void *params[1]; - - next = f->next; - params[0] = &f->arg; - reflect_call(f->ft, (void*)f->fn, 0, 0, params, nil); - f->fn = nil; - f->arg = nil; - f->next = nil; - runtime_free(f); + for(; fb; fb=next) { + next = fb->next; + for(i=0; i<(uint32)fb->cnt; i++) { + void *params[1]; + + f = &fb->fin[i]; + params[0] = &f->arg; + runtime_setblockspecial(f->arg, false); + reflect_call(f->ft, (void*)f->fn, 0, 0, params, nil); + f->fn = nil; + f->arg = nil; + } + fb->cnt = 0; + fb->next = finc; + finc = fb; } runtime_gc(1); // trigger another gc to clean up the finalized objects, if possible } @@ -783,6 +1178,9 @@ runtime_blockspecial(void *v) { uintptr *b, off, shift; + if(DebugMark) + return true; + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; @@ -791,17 +1189,23 @@ runtime_blockspecial(void *v) } void -runtime_setblockspecial(void *v) +runtime_setblockspecial(void *v, bool s) { uintptr *b, off, shift, bits, obits; + if(DebugMark) + return; + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; for(;;) { obits = *b; - bits = obits | (bitSpecial<<shift); + if(s) + bits = obits | (bitSpecial<<shift); + else + bits = obits & ~(bitSpecial<<shift); if(runtime_singleproc) { *b = bits; break; @@ -812,7 +1216,7 @@ runtime_setblockspecial(void *v) } } } - + void runtime_MHeap_MapBits(MHeap *h) { @@ -823,7 +1227,7 @@ runtime_MHeap_MapBits(MHeap *h) bitmapChunk = 8192 }; uintptr n; - + n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; n = (n+bitmapChunk-1) & ~(bitmapChunk-1); if(h->bitmap_mapped >= n) diff --git a/libgo/runtime/mheap.c b/libgo/runtime/mheap.c index cacac7d..a49b405 100644 --- a/libgo/runtime/mheap.c +++ b/libgo/runtime/mheap.c @@ -13,6 +13,7 @@ // and heapmap(i) == span for all s->start <= i < s->start+s->npages. #include "runtime.h" +#include "arch.h" #include "malloc.h" static MSpan *MHeap_AllocLocked(MHeap*, uintptr, int32); @@ -102,6 +103,7 @@ HaveSpan: runtime_throw("MHeap_AllocLocked - bad npages"); runtime_MSpanList_Remove(s); s->state = MSpanInUse; + mstats.heap_idle -= s->npages<<PageShift; if(s->npages > npage) { // Trim extra and put it back in the heap. @@ -277,6 +279,7 @@ MHeap_FreeLocked(MHeap *h, MSpan *s) // runtime_printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref); runtime_throw("MHeap_FreeLocked - invalid free"); } + mstats.heap_idle += s->npages<<PageShift; s->state = MSpanFree; runtime_MSpanList_Remove(s); sp = (uintptr*)(s->start<<PageShift); diff --git a/libgo/runtime/mprof.goc b/libgo/runtime/mprof.goc index d87be42..23c4f90 100644 --- a/libgo/runtime/mprof.goc +++ b/libgo/runtime/mprof.goc @@ -7,6 +7,7 @@ package runtime #include "runtime.h" +#include "arch.h" #include "malloc.h" #include "defs.h" #include "go-type.h" diff --git a/libgo/runtime/msize.c b/libgo/runtime/msize.c index 6e82885..e2672b0 100644 --- a/libgo/runtime/msize.c +++ b/libgo/runtime/msize.c @@ -26,6 +26,7 @@ // TODO(rsc): Compute max waste for any given size. #include "runtime.h" +#include "arch.h" #include "malloc.h" int32 runtime_class_to_size[NumSizeClasses]; diff --git a/libgo/runtime/proc.c b/libgo/runtime/proc.c index 521bcd64..8af6935 100644 --- a/libgo/runtime/proc.c +++ b/libgo/runtime/proc.c @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "arch.h" #include "malloc.h" /* so that acid generated from proc.c includes malloc data structures */ typedef struct Sched Sched; diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index ddc99eb..2767dd8 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -136,7 +136,7 @@ bool __go_sigsend(int32 sig); int64 runtime_nanotime(void); void runtime_stoptheworld(void); -void runtime_starttheworld(void); +void runtime_starttheworld(bool); void __go_go(void (*pfn)(void*), void*); void __go_gc_goroutine_init(void*); void __go_enable_gc(void); @@ -184,18 +184,21 @@ void runtime_notewakeup(Note*); MCache* runtime_allocmcache(void); void free(void *v); struct __go_func_type; -void runtime_addfinalizer(void*, void(*fn)(void*), const struct __go_func_type *); -void runtime_walkfintab(void (*fn)(void*), void (*scan)(byte *, int64)); +bool runtime_addfinalizer(void*, void(*fn)(void*), const struct __go_func_type *); #define runtime_mmap mmap #define runtime_munmap(p, s) munmap((p), (s)) #define runtime_cas(pval, old, new) __sync_bool_compare_and_swap (pval, old, new) #define runtime_casp(pval, old, new) __sync_bool_compare_and_swap (pval, old, new) +#define runtime_xadd(p, v) __sync_add_and_fetch (p, v) void runtime_sigprof(uint8 *pc, uint8 *sp, uint8 *lr); void runtime_cpuprofinit(void); void runtime_resetcpuprofiler(int32); void runtime_setcpuprofilerate(void(*)(uintptr*, int32), int32); uint32 runtime_fastrand1(void); +void runtime_procyield(uint32); +void runtime_osyield(void); +void runtime_usleep(uint32); struct __go_func_type; void reflect_call(const struct __go_func_type *, const void *, _Bool, _Bool, diff --git a/libgo/runtime/sigqueue.goc b/libgo/runtime/sigqueue.goc index 2e47222..3a90868 100644 --- a/libgo/runtime/sigqueue.goc +++ b/libgo/runtime/sigqueue.goc @@ -39,6 +39,7 @@ package runtime #include "config.h" #include "runtime.h" +#include "arch.h" #include "malloc.h" #include "defs.h" diff --git a/libgo/runtime/thread.c b/libgo/runtime/thread.c index bac3f7d..822d5da 100644 --- a/libgo/runtime/thread.c +++ b/libgo/runtime/thread.c @@ -14,19 +14,6 @@ runtime_initlock(Lock *l) runtime_throw("sem_init failed"); } -static uint32 -runtime_xadd(uint32 volatile *val, int32 delta) -{ - uint32 oval, nval; - - for(;;){ - oval = *val; - nval = oval + delta; - if(runtime_cas(val, oval, nval)) - return nval; - } -} - // noinline so that runtime_lock doesn't have to split the stack. static void runtime_lock_full(Lock *l) __attribute__ ((noinline)); diff --git a/libgo/runtime/yield.c b/libgo/runtime/yield.c new file mode 100644 index 0000000..3ebc4a4 --- /dev/null +++ b/libgo/runtime/yield.c @@ -0,0 +1,54 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define _GNU_SOURCE + +#include "config.h" + +#include <stddef.h> +#include <sys/types.h> +#include <sys/time.h> +#include <pthread.h> +#include <unistd.h> + +#ifdef HAVE_SYS_SELECT_H +#include <sys/select.h> +#endif + +#include "runtime.h" + +/* Spin wait. */ + +void +runtime_procyield (uint32 cnt) +{ + volatile uint32 i; + + for (i = 0; i < cnt; ++i) + { +#if defined (__i386__) || defined (__x86_64__) + __builtin_ia32_pause (); +#endif + } +} + +/* Ask the OS to reschedule this thread. */ + +void +runtime_osyield (void) +{ + pthread_yield (); +} + +/* Sleep for some number of microseconds. */ + +void +runtime_usleep (uint32 us) +{ + struct timeval tv; + + tv.tv_sec = us / 1000000; + tv.tv_usec = us % 1000000; + select (0, NULL, NULL, NULL, &tv); +} diff --git a/libgo/testsuite/gotest b/libgo/testsuite/gotest index 69cf8ee..1dedaa8 100755 --- a/libgo/testsuite/gotest +++ b/libgo/testsuite/gotest @@ -331,6 +331,10 @@ localname() { pattern='Benchmark([^a-z].*)?' benchmarks=$($NM -p -v _gotest_.o $xofile | egrep ' T .*\.'$pattern'$' | grep -v '\..*\..*\.' | sed 's/.* //' | sed 's/.*\.\(.*\.\)/\1/') + # examples are named ExampleFoo + pattern='Example([^a-z].*)?' + examples=$($NM -p -v _gotest_.o $xofile | egrep ' T .*\.'$pattern'$' | grep -v '\..*\..*\.' | sed 's/.* //' | sed 's/.*\.\(.*\.\)/\1/') + # package spec echo 'package main' echo @@ -353,6 +357,7 @@ localname() { echo ' {"'$i'", '$j'},' done echo '}' + # benchmark array # The comment makes the multiline declaration # gofmt-safe even when there are no benchmarks. @@ -363,6 +368,17 @@ localname() { echo ' {"'$i'", '$j'},' done echo '}' + + # examples array + echo 'var examples = []testing.InternalExample{ //' + # This doesn't work because we don't pick up the output. + #for i in $examples + #do + # j=$(localname $i) + # echo ' {"'$i'", '$j', ""},' + #done + echo '}' + # body echo \ ' @@ -381,7 +397,7 @@ func matchString(pat, str string) (result bool, err __os__.Error) { } func main() { - testing.Main(matchString, tests, benchmarks) + testing.Main(matchString, tests, benchmarks, examples) }' }>_testmain.go |
