From 54bbd1e59317a6e9658eb8098657078cc8e81979 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 14 Aug 2022 20:52:06 -0600 Subject: wip: Reduce test differences [ci-skip] - Handle UTF-16 surrogate pairs - Handle cycles in values - Handle cycles in types - Better errors - Handle case-folding of struct field names - Allow []byteTypeWithMethods - Fix struct field-order - Fix handling of interfaces storing pointers - Enforce a maximum decode depth - Validate struct tags --- compat/json/borrowed_decode_test.go | 37 ++++++++++++----------------- compat/json/borrowed_encode_test.go | 15 ++++-------- compat/json/borrowed_misc.go | 14 +++++++++++ compat/json/borrowed_scanner_test.go | 4 ---- compat/json/borrowed_tagkey_test.go | 1 - compat/json/compat.go | 45 +++++++++++++++++++++++++++++++++--- compat/json/compat_test.go | 6 +++++ 7 files changed, 82 insertions(+), 40 deletions(-) create mode 100644 compat/json/borrowed_misc.go (limited to 'compat') diff --git a/compat/json/borrowed_decode_test.go b/compat/json/borrowed_decode_test.go index 306f85e..4b84718 100644 --- a/compat/json/borrowed_decode_test.go +++ b/compat/json/borrowed_decode_test.go @@ -455,8 +455,8 @@ var unmarshalTests = []unmarshalTest{ {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, - {in: `[2, 3`, err: &SyntaxError{Err: "unexpected end of JSON input", Offset: 5}}, // MODIFIED - {in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{Err: "invalid character '}' in numeric literal", Offset: 9}}, // MODIFIED + {in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}}, + {in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}}, // raw value errors {in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, @@ -957,7 +957,7 @@ var unmarshalTests = []unmarshalTest{ in: `invalid`, ptr: new(Number), err: &SyntaxError{ - Err: "invalid character 'i' looking for beginning of value", // MODIFIED + msg: "invalid character 'i' looking for beginning of value", Offset: 1, }, }, @@ -1040,7 +1040,6 @@ func TestMarshalNumberZeroVal(t *testing.T) { } func TestMarshalEmbeds(t *testing.T) { - t.Skip() // TODO top := &Top{ Level0: 1, Embed0: Embed0{ @@ -1089,17 +1088,16 @@ func equalError(a, b error) bool { if b == nil { return a == nil } - return true // a.Error() == b.Error() // MODIFIED + return a.Error() == b.Error() } func TestUnmarshal(t *testing.T) { - t.Skip() // TODO for i, tt := range unmarshalTests { scan := lowmemjson.ReEncoder{Out: io.Discard} // MODIFIED in := []byte(tt.in) - if _, err := scan.Write(in); err != nil { + if err := checkValid(in, &scan); err != nil { if !equalError(err, tt.err) { - t.Errorf("#%d: checkValid: %#v\n\n%s", i, err, tt.in) + t.Errorf("#%d: checkValid: %#v", i, err) continue } } @@ -1142,11 +1140,11 @@ func TestUnmarshal(t *testing.T) { continue } if !reflect.DeepEqual(v.Elem().Interface(), tt.out) { - t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v\n\n%s", i, v.Elem().Interface(), tt.out, tt.in) + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), tt.out) data, _ := Marshal(v.Elem().Interface()) - println(string(data)) + t.Log(string(data)) // MODIFIED data, _ = Marshal(tt.out) - println(string(data)) + t.Log(string(data)) // MODIFIED continue } @@ -1311,7 +1309,7 @@ func TestErrorMessageFromMisusedString(t *testing.T) { var s WrongString err := NewDecoder(r).Decode(&s) got := fmt.Sprintf("%v", err) - if err == nil { // if got != tt.err { // MODIFIED + if got != tt.err { t.Errorf("%d. got err = %q, want %q", n, got, tt.err) } } @@ -1742,7 +1740,6 @@ var interfaceSetTests = []struct { } func TestInterfaceSet(t *testing.T) { - t.Skip() // TODO for _, tt := range interfaceSetTests { b := struct{ X any }{tt.pre} blob := `{"X":` + tt.json + `}` @@ -2015,7 +2012,7 @@ var decodeTypeErrorTests = []struct { func TestUnmarshalTypeError(t *testing.T) { for _, item := range decodeTypeErrorTests { err := Unmarshal([]byte(item.src), item.dest) - if err == nil { // if _, ok := err.(*UnmarshalTypeError); !ok { // MODIFIED + if _, ok := err.(*UnmarshalTypeError); !ok { t.Errorf("expected type error for Unmarshal(%q, type %T): got %T", item.src, item.dest, err) } @@ -2037,7 +2034,7 @@ func TestUnmarshalSyntax(t *testing.T) { var x any for _, src := range unmarshalSyntaxTests { err := Unmarshal([]byte(src), &x) - if err == nil { // _, ok := err.(*SyntaxError); !ok { // MODIFIED + if _, ok := err.(*SyntaxError); !ok { t.Errorf("expected syntax error for Unmarshal(%q): got %T", src, err) } } @@ -2202,9 +2199,9 @@ func TestInvalidUnmarshalText(t *testing.T) { t.Errorf("Unmarshal expecting error, got nil") continue } - // if got := err.Error(); got != tt.want { // MODIFIED - // t.Errorf("Unmarshal = %q; want %q", got, tt.want) // MODIFIED - // } // MODIFIED + if got := err.Error(); got != tt.want { + t.Errorf("Unmarshal = %q; want %q", got, tt.want) + } } } @@ -2243,7 +2240,6 @@ func TestInvalidStringOption(t *testing.T) { // (Issue 28145) If the embedded struct is given an explicit name and has // exported methods, don't cause a panic trying to get its value. func TestUnmarshalEmbeddedUnexported(t *testing.T) { - t.Skip() // TODO type ( embed1 struct{ Q int } embed2 struct{ Q int } @@ -2365,7 +2361,6 @@ func TestUnmarshalEmbeddedUnexported(t *testing.T) { } func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) { - t.Skip() // TODO tests := []struct { in string err error @@ -2417,7 +2412,6 @@ func TestUnmarshalPanic(t *testing.T) { // The decoder used to hang if decoding into an interface pointing to its own address. // See golang.org/issues/31740. func TestUnmarshalRecursivePointer(t *testing.T) { - t.Skip() // TODO var v any v = &v data := []byte(`{"a": "b"}`) @@ -2493,7 +2487,6 @@ func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { } func TestUnmarshalMaxDepth(t *testing.T) { - t.Skip() // TODO testcases := []struct { name string data string diff --git a/compat/json/borrowed_encode_test.go b/compat/json/borrowed_encode_test.go index bb7c9dc..11c2db4 100644 --- a/compat/json/borrowed_encode_test.go +++ b/compat/json/borrowed_encode_test.go @@ -226,11 +226,11 @@ var unsupportedValues = []any{ math.NaN(), math.Inf(-1), math.Inf(1), - //pointerCycle, // MODIFIED - //pointerCycleIndirect, // MODIFIED - //mapCycle, // MODIFIED - //sliceCycle, // MODIFIED - //recursiveSliceCycle, // MODIFIED + pointerCycle, + pointerCycleIndirect, + mapCycle, + sliceCycle, + recursiveSliceCycle, } func TestUnsupportedValues(t *testing.T) { @@ -344,7 +344,6 @@ func (CText) MarshalText() ([]byte, error) { } func TestMarshalerEscaping(t *testing.T) { - t.Skip() // MODIFIED var c C want := `"\u003c\u0026\u003e"` b, err := Marshal(c) @@ -877,7 +876,6 @@ func (f textfloat) MarshalText() ([]byte, error) { return tenc(`TF:%0.2f`, f) } // Issue 13783 func TestEncodeBytekind(t *testing.T) { - t.Skip() // TODO testdata := []struct { data any want string @@ -1139,7 +1137,6 @@ func TestMarshalRawMessageValue(t *testing.T) { if err != nil { t.Errorf("test %d, unexpected failure: %v", i, err) } else { - t.Skip() // MODIFIED t.Errorf("test %d, unexpected success", i) } } @@ -1178,7 +1175,6 @@ func TestMarshalUncommonFieldNames(t *testing.T) { } } -/* // MODIFIED func TestMarshalerError(t *testing.T) { s := "test variable" st := reflect.TypeOf(s) @@ -1205,4 +1201,3 @@ func TestMarshalerError(t *testing.T) { } } } -*/ // MODIFIED diff --git a/compat/json/borrowed_misc.go b/compat/json/borrowed_misc.go new file mode 100644 index 0000000..30a3b0e --- /dev/null +++ b/compat/json/borrowed_misc.go @@ -0,0 +1,14 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +// A SyntaxError is a description of a JSON syntax error. +// Unmarshal will return a SyntaxError if the JSON can't be parsed. +type SyntaxError struct { + msg string // description of error + Offset int64 // error occurred after reading Offset bytes +} + +func (e *SyntaxError) Error() string { return e.msg } diff --git a/compat/json/borrowed_scanner_test.go b/compat/json/borrowed_scanner_test.go index 4955405..3474b3e 100644 --- a/compat/json/borrowed_scanner_test.go +++ b/compat/json/borrowed_scanner_test.go @@ -65,7 +65,6 @@ var ex1i = `[ ]` func TestCompact(t *testing.T) { - t.Skip() // TODO var buf bytes.Buffer for _, tt := range examples { buf.Reset() @@ -86,7 +85,6 @@ func TestCompact(t *testing.T) { } func TestCompactSeparators(t *testing.T) { - t.Skip() // TODO // U+2028 and U+2029 should be escaped inside strings. // They should not appear outside strings. tests := []struct { @@ -106,7 +104,6 @@ func TestCompactSeparators(t *testing.T) { } func TestIndent(t *testing.T) { - t.Skip() // TODO var buf bytes.Buffer for _, tt := range examples { buf.Reset() @@ -192,7 +189,6 @@ var indentErrorTests = []indentErrorTest{ } func TestIndentErrors(t *testing.T) { - t.Skip() // TODO for i, tt := range indentErrorTests { slice := make([]uint8, 0) buf := bytes.NewBuffer(slice) diff --git a/compat/json/borrowed_tagkey_test.go b/compat/json/borrowed_tagkey_test.go index 6a2d612..6330efd 100644 --- a/compat/json/borrowed_tagkey_test.go +++ b/compat/json/borrowed_tagkey_test.go @@ -96,7 +96,6 @@ var structTagObjectKeyTests = []struct { } func TestStructTagObjectKey(t *testing.T) { - t.Skip() // TODO for _, tt := range structTagObjectKeyTests { b, err := Marshal(tt.raw) if err != nil { diff --git a/compat/json/compat.go b/compat/json/compat.go index 78a9d5f..b26914b 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -8,7 +8,9 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "io" + "strconv" "git.lukeshu.com/go/lowmemjson" ) @@ -19,7 +21,7 @@ type ( RawMessage = json.RawMessage // low-level decode errors - SyntaxError = lowmemjson.SyntaxError + //SyntaxError = lowmemjson.DecodeSyntaxError // expose a field UnmarshalFieldError = json.UnmarshalFieldError UnmarshalTypeError = json.UnmarshalTypeError // lowmemjson.DecodeTypeError @@ -28,7 +30,7 @@ type ( // marshal errors InvalidUTF8Error = json.InvalidUTF8Error - MarshalerError = json.MarshalerError + MarshalerError = lowmemjson.EncodeMethodError // expose a field UnsupportedTypeError = json.UnsupportedTypeError UnsupportedValueError = json.UnsupportedValueError ) @@ -92,7 +94,7 @@ func Valid(data []byte) bool { } func Unmarshal(data []byte, ptr any) error { - return lowmemjson.Decode(bytes.NewReader(data), ptr) + return NewDecoder(bytes.NewReader(data)).Decode(ptr) } ///////////////////////////////////////////////////////////////////// @@ -113,6 +115,43 @@ func NewDecoder(r io.Reader) *Decoder { } } +func (dec *Decoder) Decode(ptr any) error { + err := dec.Decoder.Decode(ptr) + if derr, ok := err.(*lowmemjson.DecodeError); ok { + switch terr := derr.Err.(type) { + case *lowmemjson.DecodeSyntaxError: + err = &SyntaxError{ + msg: terr.Err.Error(), + Offset: terr.Offset, + } + case *lowmemjson.DecodeTypeError: + if typeErr, ok := terr.Err.(*json.UnmarshalTypeError); ok { + err = &UnmarshalTypeError{ + Value: typeErr.Value, + Type: typeErr.Type, + Offset: typeErr.Offset, + Struct: derr.FieldParent, + Field: derr.FieldName, + } + } else if _, isArgErr := terr.Err.(*lowmemjson.DecodeArgumentError); terr.Err != nil && + !isArgErr && + !errors.Is(terr.Err, strconv.ErrSyntax) && + !errors.Is(terr.Err, strconv.ErrRange) { + err = terr.Err + } else { + err = &UnmarshalTypeError{ + Value: terr.JSONType, + Type: terr.GoType, + Offset: terr.Offset, + Struct: derr.FieldParent, + Field: derr.FieldName, + } + } + } + } + return err +} + func (dec *Decoder) Buffered() io.Reader { dat, _ := dec.buf.Peek(dec.buf.Buffered()) return bytes.NewReader(dat) diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 399ff02..5a34d22 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -7,11 +7,17 @@ package json import ( "bytes" + "git.lukeshu.com/go/lowmemjson" "git.lukeshu.com/go/lowmemjson/internal" ) var parseTag = internal.ParseTag +func checkValid(in []byte, scan *lowmemjson.ReEncoder) error { + _, err := scan.Write(in) + return err +} + const ( startDetectingCyclesAfter = 1000 ) -- cgit v1.2.3-2-g168b