From a4ffa83919375653415b670fdbf136e29c1b1c2c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 23 Feb 2023 22:08:27 -0700 Subject: compat/json: Unify the error conversion --- compat/json/compat.go | 168 ++++++++++++++++++++++++++++---------------------- 1 file changed, 93 insertions(+), 75 deletions(-) (limited to 'compat') diff --git a/compat/json/compat.go b/compat/json/compat.go index 695c1a8..c2d47c0 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -40,22 +40,100 @@ type ( // MarshalerError = json.MarshalerError // Duplicated to access a private field. ) -// Encode wrappers /////////////////////////////////////////////////// +// Error conversion ////////////////////////////////////////////////// -func convertEncodeError(err error) error { - if me, ok := err.(*lowmemjson.EncodeMethodError); ok { - err = &MarshalerError{ - Type: me.Type, - Err: me.Err, - sourceFunc: me.SourceFunc, +func convertError(err error, isUnmarshal bool) error { + switch err := err.(type) { + case nil: + return nil + case *lowmemjson.DecodeArgumentError: + return err + case *lowmemjson.DecodeError: + switch suberr := err.Err.(type) { + case *lowmemjson.DecodeReadError: + return err + case *lowmemjson.DecodeSyntaxError: + switch { + case errors.Is(err, io.EOF): + return io.EOF + case errors.Is(err, io.ErrUnexpectedEOF) && isUnmarshal: + return &SyntaxError{ + msg: "unexpected end of JSON input", + Offset: suberr.Offset, + } + default: + return &SyntaxError{ + msg: suberr.Err.Error(), + Offset: suberr.Offset + 1, + } + } + case *lowmemjson.DecodeTypeError: + switch subsuberr := suberr.Err.(type) { + case *UnmarshalTypeError: + // Populate the .Struct and .Field members. + subsuberr.Struct = err.FieldParent + subsuberr.Field = err.FieldName + return subsuberr + default: + switch { + case errors.Is(err, lowmemjson.ErrDecodeNonEmptyInterface), + errors.Is(err, strconv.ErrSyntax), + errors.Is(err, strconv.ErrRange): + return &UnmarshalTypeError{ + Value: suberr.JSONType, + Type: suberr.GoType, + Offset: suberr.Offset, + Struct: err.FieldParent, + Field: err.FieldName, + } + default: + return subsuberr + } + case nil, *lowmemjson.DecodeArgumentError: + return &UnmarshalTypeError{ + Value: suberr.JSONType, + Type: suberr.GoType, + Offset: suberr.Offset, + Struct: err.FieldParent, + Field: err.FieldName, + } + } + default: + panic(fmt.Errorf("should not happen: unexpected lowmemjson.DecodeError sub-type: %T: %w", suberr, err)) + } + case *lowmemjson.EncodeWriteError: + return err + case *lowmemjson.EncodeTypeError: + return err + case *lowmemjson.EncodeValueError: + return err + case *lowmemjson.EncodeMethodError: + return &MarshalerError{ + Type: err.Type, + Err: err.Err, + sourceFunc: err.SourceFunc, + } + case *lowmemjson.ReEncodeWriteError: + return err + case *lowmemjson.ReEncodeSyntaxError: + ret := &SyntaxError{ + msg: err.Err.Error(), + Offset: err.Offset + 1, } + if errors.Is(err, io.ErrUnexpectedEOF) { + ret.msg = "unexpected end of JSON input" + } + return ret + default: + panic(fmt.Errorf("should not happen: unexpected lowmemjson error type: %T: %w", err, err)) } - return err } +// Encode wrappers /////////////////////////////////////////////////// + func marshal(v any, cfg lowmemjson.ReEncoderConfig) ([]byte, error) { var buf bytes.Buffer - if err := convertEncodeError(lowmemjson.NewEncoder(lowmemjson.NewReEncoder(&buf, cfg)).Encode(v)); err != nil { + if err := convertError(lowmemjson.NewEncoder(lowmemjson.NewReEncoder(&buf, cfg)).Encode(v), false); err != nil { return nil, err } return buf.Bytes(), nil @@ -105,7 +183,7 @@ func (enc *Encoder) refreshConfig() { } func (enc *Encoder) Encode(v any) error { - if err := convertEncodeError(enc.encoder.Encode(v)); err != nil { + if err := convertError(enc.encoder.Encode(v), false); err != nil { enc.buf.Reset() return err } @@ -133,19 +211,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) { // ReEncode wrappers ///////////////////////////////////////////////// -func convertReEncodeError(err error) error { - if se, ok := err.(*lowmemjson.ReEncodeSyntaxError); ok { - err = &SyntaxError{ - msg: se.Err.Error(), - Offset: se.Offset + 1, - } - if errors.Is(se.Err, io.ErrUnexpectedEOF) { - err.(*SyntaxError).msg = "unexpected end of JSON input" - } - } - return err -} - func HTMLEscape(dst *bytes.Buffer, src []byte) { for n := 0; n < len(src); { c, size := utf8.DecodeRune(src[n:]) @@ -172,7 +237,7 @@ func reencode(dst io.Writer, src []byte, cfg lowmemjson.ReEncoderConfig) error { if err == nil { err = formatter.Close() } - return convertReEncodeError(err) + return convertError(err, false) } func Compact(dst *bytes.Buffer, src []byte) error { @@ -237,53 +302,6 @@ func Valid(data []byte) bool { // Decode wrappers /////////////////////////////////////////////////// -func convertDecodeError(err error, isUnmarshal bool) error { - if derr, ok := err.(*lowmemjson.DecodeError); ok { - switch terr := derr.Err.(type) { - case *lowmemjson.DecodeSyntaxError: - switch { - case errors.Is(terr.Err, io.EOF): - err = io.EOF - case errors.Is(terr.Err, io.ErrUnexpectedEOF) && isUnmarshal: - err = &SyntaxError{ - msg: "unexpected end of JSON input", - Offset: terr.Offset, - } - default: - err = &SyntaxError{ - msg: terr.Err.Error(), - Offset: terr.Offset + 1, - } - } - case *lowmemjson.DecodeTypeError: - if typeErr, ok := terr.Err.(*json.UnmarshalTypeError); ok { - err = &UnmarshalTypeError{ - Value: typeErr.Value, - Type: typeErr.Type, - Offset: typeErr.Offset, - Struct: derr.FieldParent, - Field: derr.FieldName, - } - } else if _, isArgErr := terr.Err.(*lowmemjson.DecodeArgumentError); terr.Err != nil && - !isArgErr && - !errors.Is(terr.Err, lowmemjson.ErrDecodeNonEmptyInterface) && - !errors.Is(terr.Err, strconv.ErrSyntax) && - !errors.Is(terr.Err, strconv.ErrRange) { - err = terr.Err - } else { - err = &UnmarshalTypeError{ - Value: terr.JSONType, - Type: terr.GoType, - Offset: terr.Offset, - Struct: derr.FieldParent, - Field: derr.FieldName, - } - } - } - } - return err -} - type decodeValidator struct{} func (*decodeValidator) DecodeJSON(r io.RuneScanner) error { @@ -301,10 +319,10 @@ func (*decodeValidator) DecodeJSON(r io.RuneScanner) error { var _ lowmemjson.Decodable = (*decodeValidator)(nil) func Unmarshal(data []byte, ptr any) error { - if err := convertDecodeError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(&decodeValidator{}), true); err != nil { + if err := convertError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(&decodeValidator{}), true); err != nil { return err } - if err := convertDecodeError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(ptr), true); err != nil { + if err := convertError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(ptr), true); err != nil { return err } return nil @@ -363,10 +381,10 @@ func NewDecoder(r io.Reader) *Decoder { } func (dec *Decoder) Decode(ptr any) error { - if err := convertDecodeError(dec.validator.Decode(&decodeValidator{}), false); err != nil { + if err := convertError(dec.validator.Decode(&decodeValidator{}), false); err != nil { return err } - if err := convertDecodeError(dec.Decoder.Decode(ptr), false); err != nil { + if err := convertError(dec.Decoder.Decode(ptr), false); err != nil { return err } return nil -- cgit v1.2.3-2-g168b From d35495540df2b6d3ba16c84ce21627d9dbae000c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 10 Feb 2023 23:38:26 -0700 Subject: Fuzz for equivalence between stdlib and lowmemjson --- compat/json/equiv_test.go | 160 +++++++++++++++++++++ .../json/testdata/fuzz/FuzzEquiv/0064ebc3507e959b | 2 + .../json/testdata/fuzz/FuzzEquiv/19981bffc2abbaf1 | 2 + .../json/testdata/fuzz/FuzzEquiv/57365320c0968611 | 2 + .../json/testdata/fuzz/FuzzEquiv/5cd6893f25481dae | 2 + .../json/testdata/fuzz/FuzzEquiv/6a6612e05e0f9e32 | 2 + .../json/testdata/fuzz/FuzzEquiv/77e6e971d8684f84 | 2 + .../json/testdata/fuzz/FuzzEquiv/8727b16d337d7b81 | 2 + .../json/testdata/fuzz/FuzzEquiv/96aac43014471adc | 2 + .../json/testdata/fuzz/FuzzEquiv/9cc52906ed53ef5f | 2 + .../json/testdata/fuzz/FuzzEquiv/a0b9ecf4e99fd85d | 2 + .../json/testdata/fuzz/FuzzEquiv/a5775dd298b90a6c | 2 + .../json/testdata/fuzz/FuzzEquiv/af9bedcb9e0a31e8 | 2 + .../json/testdata/fuzz/FuzzEquiv/f6b0960dd3331a00 | 2 + .../json/testdata/fuzz/FuzzEquiv/fbbce5ea61559cc6 | 2 + .../json/testdata/fuzz/FuzzEquiv/fd29ccbb2af92d4f | 2 + 16 files changed, 190 insertions(+) create mode 100644 compat/json/equiv_test.go create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/0064ebc3507e959b create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/19981bffc2abbaf1 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/57365320c0968611 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/5cd6893f25481dae create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/6a6612e05e0f9e32 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/77e6e971d8684f84 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/8727b16d337d7b81 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/96aac43014471adc create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/9cc52906ed53ef5f create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/a0b9ecf4e99fd85d create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/a5775dd298b90a6c create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/af9bedcb9e0a31e8 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/f6b0960dd3331a00 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/fbbce5ea61559cc6 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/fd29ccbb2af92d4f (limited to 'compat') diff --git a/compat/json/equiv_test.go b/compat/json/equiv_test.go new file mode 100644 index 0000000..246e4b3 --- /dev/null +++ b/compat/json/equiv_test.go @@ -0,0 +1,160 @@ +// Copyright (C) 2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package json_test + +import ( + "bytes" + std "encoding/json" + "errors" + "io" + "strconv" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + + low "git.lukeshu.com/go/lowmemjson/compat/json" +) + +func assertEquivErr(t *testing.T, stdErr, lowErr error) { + if (stdErr == nil) || (lowErr == nil) { + // Nil-equal. + assert.Equal(t, stdErr, lowErr) + return + } + switch stdErr.(type) { + case *std.SyntaxError: + if lowErr != nil { + stdMsg := stdErr.Error() + lowMsg := lowErr.Error() + + // https://github.com/golang/go/issues/58680 + if strings.HasPrefix(stdMsg, `invalid character ' ' `) && + (errors.Is(lowErr, io.ErrUnexpectedEOF) || lowMsg == "unexpected end of JSON input") { + return + } + + // https://github.com/golang/go/issues/58713 + prefix := `invalid character '` + if stdMsg != lowMsg && strings.HasPrefix(stdMsg, prefix) && strings.HasPrefix(lowMsg, prefix) { + stdRune, stdRuneSize := utf8.DecodeRuneInString(stdMsg[len(prefix):]) + lowByte := lowMsg[len(prefix)] + if lowByte == '\\' { + switch lowMsg[len(prefix)+1] { + case 'u': + lowRune, _ := strconv.ParseUint(lowMsg[len(prefix)+2:][:4], 16, 32) + var buf [4]byte + utf8.EncodeRune(buf[:], rune(lowRune)) + lowByte = buf[0] + case 'U': + lowRune, _ := strconv.ParseUint(lowMsg[len(prefix)+2:][:8], 16, 32) + var buf [4]byte + utf8.EncodeRune(buf[:], rune(lowRune)) + lowByte = buf[0] + } + } + if stdRune == rune(lowByte) { + lowRuneStr := lowMsg[len(prefix):] + lowRuneStr = lowRuneStr[:strings.IndexByte(lowRuneStr, '\'')] + stdMsg = prefix + lowRuneStr + stdMsg[len(prefix)+stdRuneSize:] + stdErr = errors.New(stdMsg) + } + } + } + // Text-equal. + assert.Equal(t, stdErr.Error(), lowErr.Error()) + // TODO: Assert that they are deep-equal (but be permissive of these not being type aliases). + case *std.MarshalerError: + // Text-equal. + assert.Equal(t, stdErr.Error(), lowErr.Error()) + // TODO: Assert that they are deep-equal (but be permissive of these not being type aliases). + default: + // Text-equal. + assert.Equal(t, stdErr.Error(), lowErr.Error()) + // TODO: Assert that they are deep-equal. + } +} + +func FuzzEquiv(f *testing.F) { + f.Fuzz(func(t *testing.T, str []byte) { + t.Logf("str=%q", str) + t.Run("HTMLEscape", func(t *testing.T) { + var stdOut bytes.Buffer + std.HTMLEscape(&stdOut, str) + + var lowOut bytes.Buffer + low.HTMLEscape(&lowOut, str) + + assert.Equal(t, stdOut.String(), lowOut.String()) + }) + t.Run("Compact", func(t *testing.T) { + var stdOut bytes.Buffer + stdErr := std.Compact(&stdOut, str) + + var lowOut bytes.Buffer + lowErr := low.Compact(&lowOut, str) + + assert.Equal(t, stdOut.String(), lowOut.String()) + assertEquivErr(t, stdErr, lowErr) + }) + t.Run("Indent", func(t *testing.T) { + var stdOut bytes.Buffer + stdErr := std.Indent(&stdOut, str, "»", "\t") + + var lowOut bytes.Buffer + lowErr := low.Indent(&lowOut, str, "»", "\t") + + assert.Equal(t, stdOut.String(), lowOut.String()) + assertEquivErr(t, stdErr, lowErr) + }) + t.Run("Valid", func(t *testing.T) { + stdValid := std.Valid(str) && utf8.Valid(str) // https://github.com/golang/go/issues/58517 + lowValid := low.Valid(str) + assert.Equal(t, stdValid, lowValid) + }) + t.Run("Decode-Encode", func(t *testing.T) { + var stdObj any + stdErr := std.NewDecoder(bytes.NewReader(str)).Decode(&stdObj) + + var lowObj any + lowErr := low.NewDecoder(bytes.NewReader(str)).Decode(&lowObj) + + assert.Equal(t, stdObj, lowObj) + assertEquivErr(t, stdErr, lowErr) + if t.Failed() { + return + } + + var stdOut bytes.Buffer + stdErr = std.NewEncoder(&stdOut).Encode(stdObj) + + var lowOut bytes.Buffer + lowErr = low.NewEncoder(&lowOut).Encode(lowObj) + + assert.Equal(t, stdOut.String(), lowOut.String()) + assertEquivErr(t, stdErr, lowErr) + }) + t.Run("Unmarshal-Marshal", func(t *testing.T) { + var stdObj any + stdErr := std.Unmarshal(str, &stdObj) + + var lowObj any + lowErr := low.Unmarshal(str, &lowObj) + + assert.Equal(t, stdObj, lowObj) + assertEquivErr(t, stdErr, lowErr) + if t.Failed() { + return + } + + stdOut, stdErr := std.Marshal(stdObj) + lowOut, lowErr := low.Marshal(lowObj) + + assert.Equal(t, string(stdOut), string(lowOut)) + assertEquivErr(t, stdErr, lowErr) + }) + }) +} diff --git a/compat/json/testdata/fuzz/FuzzEquiv/0064ebc3507e959b b/compat/json/testdata/fuzz/FuzzEquiv/0064ebc3507e959b new file mode 100644 index 0000000..96e9e53 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/0064ebc3507e959b @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("𐠁") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/19981bffc2abbaf1 b/compat/json/testdata/fuzz/FuzzEquiv/19981bffc2abbaf1 new file mode 100644 index 0000000..ecbe8af --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/19981bffc2abbaf1 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("A") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/57365320c0968611 b/compat/json/testdata/fuzz/FuzzEquiv/57365320c0968611 new file mode 100644 index 0000000..5aace7f --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/57365320c0968611 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("[200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000]") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/5cd6893f25481dae b/compat/json/testdata/fuzz/FuzzEquiv/5cd6893f25481dae new file mode 100644 index 0000000..a51778b --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/5cd6893f25481dae @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0E00") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/6a6612e05e0f9e32 b/compat/json/testdata/fuzz/FuzzEquiv/6a6612e05e0f9e32 new file mode 100644 index 0000000..fe2e128 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/6a6612e05e0f9e32 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\uD800\"") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/77e6e971d8684f84 b/compat/json/testdata/fuzz/FuzzEquiv/77e6e971d8684f84 new file mode 100644 index 0000000..e3c530f --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/77e6e971d8684f84 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\uebae") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/8727b16d337d7b81 b/compat/json/testdata/fuzz/FuzzEquiv/8727b16d337d7b81 new file mode 100644 index 0000000..e8000f3 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/8727b16d337d7b81 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("00") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/96aac43014471adc b/compat/json/testdata/fuzz/FuzzEquiv/96aac43014471adc new file mode 100644 index 0000000..9461c7a --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/96aac43014471adc @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/9cc52906ed53ef5f b/compat/json/testdata/fuzz/FuzzEquiv/9cc52906ed53ef5f new file mode 100644 index 0000000..1edfb06 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/9cc52906ed53ef5f @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/a0b9ecf4e99fd85d b/compat/json/testdata/fuzz/FuzzEquiv/a0b9ecf4e99fd85d new file mode 100644 index 0000000..b3c523c --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/a0b9ecf4e99fd85d @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0.") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/a5775dd298b90a6c b/compat/json/testdata/fuzz/FuzzEquiv/a5775dd298b90a6c new file mode 100644 index 0000000..ca6f6f5 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/a5775dd298b90a6c @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\u") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/af9bedcb9e0a31e8 b/compat/json/testdata/fuzz/FuzzEquiv/af9bedcb9e0a31e8 new file mode 100644 index 0000000..778cc61 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/af9bedcb9e0a31e8 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0 ") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/f6b0960dd3331a00 b/compat/json/testdata/fuzz/FuzzEquiv/f6b0960dd3331a00 new file mode 100644 index 0000000..9644b51 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/f6b0960dd3331a00 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"0\x85\xcd\xc0\xf3\xcb\xc1\xb3\xf2\xf5\xa4\xc1\xd40\xba\xe9\"") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/fbbce5ea61559cc6 b/compat/json/testdata/fuzz/FuzzEquiv/fbbce5ea61559cc6 new file mode 100644 index 0000000..712fab9 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/fbbce5ea61559cc6 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\U00054516") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/fd29ccbb2af92d4f b/compat/json/testdata/fuzz/FuzzEquiv/fd29ccbb2af92d4f new file mode 100644 index 0000000..9dc2675 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/fd29ccbb2af92d4f @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("Ǒ") -- cgit v1.2.3-2-g168b From 051f966039028d257f27fc3a42c10cbff9f7c738 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 23 Feb 2023 21:30:12 -0700 Subject: decode: Include the invalid UTF-8 byte in error messages --- compat/json/compat.go | 33 ++++++++++++++-- compat/json/compat_test.go | 46 ++++++++++++---------- .../json/testdata/fuzz/FuzzEquiv/9e35149f0eb0866b | 2 + 3 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/9e35149f0eb0866b (limited to 'compat') diff --git a/compat/json/compat.go b/compat/json/compat.go index c2d47c0..6f13fbb 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -329,7 +329,10 @@ func Unmarshal(data []byte, ptr any) error { } type teeRuneScanner struct { - src io.RuneScanner + src interface { + io.RuneScanner + io.ByteScanner + } dst *bytes.Buffer lastSize int } @@ -337,11 +340,14 @@ type teeRuneScanner struct { func (tee *teeRuneScanner) ReadRune() (r rune, size int, err error) { r, size, err = tee.src.ReadRune() if err == nil { - if _, err := tee.dst.WriteRune(r); err != nil { - return 0, 0, err + if r == utf8.RuneError && size == 1 { + _ = tee.src.UnreadRune() + b, _ := tee.src.ReadByte() + _ = tee.dst.WriteByte(b) + } else { + _, _ = tee.dst.WriteRune(r) } } - tee.lastSize = size return } @@ -356,6 +362,25 @@ func (tee *teeRuneScanner) UnreadRune() error { return nil } +func (tee *teeRuneScanner) ReadByte() (b byte, err error) { + b, err = tee.src.ReadByte() + if err == nil { + _ = tee.dst.WriteByte(b) + tee.lastSize = 1 + } + return +} + +func (tee *teeRuneScanner) UnreadByte() error { + if tee.lastSize != 1 { + return lowmemjson.ErrInvalidUnreadRune + } + _ = tee.src.UnreadByte() + tee.dst.Truncate(tee.dst.Len() - tee.lastSize) + tee.lastSize = 0 + return nil +} + type Decoder struct { validatorBuf *bufio.Reader validator *lowmemjson.Decoder diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 098ac85..6aab103 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -72,13 +72,14 @@ func TestCompatCompact(t *testing.T) { Err string } testcases := map[string]testcase{ - "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, - "object": {In: `{}`, Out: `{}`}, - "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, - "float": {In: `1.200e003`, Out: `1.200e003`}, - "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, - "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, - "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, + "float": {In: `1.200e003`, Out: `1.200e003`}, + "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, + "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, + "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, + "invalid-utf8": {In: "\x85", Err: `invalid character '\u0085' looking for beginning of value`}, } for tcName, tc := range testcases { tc := tc @@ -105,20 +106,21 @@ func TestCompatIndent(t *testing.T) { Err string } testcases := map[string]testcase{ - "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, - "object": {In: `{}`, Out: `{}`}, - "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, - "float": {In: `1.200e003`, Out: `1.200e003`}, - "tailws0": {In: `0`, Out: `0`}, - "tailws1": {In: `0 `, Out: `0 `}, - "tailws2": {In: `0 `, Out: `0 `}, - "tailws3": {In: "0\n", Out: "0\n"}, - "headws1": {In: ` 0`, Out: `0`}, - "objws1": {In: `{"a" : 1}`, Out: "{\n>.\"a\": 1\n>}"}, - "objws2": {In: "{\"a\"\n:\n1}", Out: "{\n>.\"a\": 1\n>}"}, - "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, - "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, - "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, + "float": {In: `1.200e003`, Out: `1.200e003`}, + "tailws0": {In: `0`, Out: `0`}, + "tailws1": {In: `0 `, Out: `0 `}, + "tailws2": {In: `0 `, Out: `0 `}, + "tailws3": {In: "0\n", Out: "0\n"}, + "headws1": {In: ` 0`, Out: `0`}, + "objws1": {In: `{"a" : 1}`, Out: "{\n>.\"a\": 1\n>}"}, + "objws2": {In: "{\"a\"\n:\n1}", Out: "{\n>.\"a\": 1\n>}"}, + "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, + "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, + "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, + "invalid-utf8": {In: "\x85", Err: `invalid character '\u0085' looking for beginning of value`}, } for tcName, tc := range testcases { tc := tc @@ -181,6 +183,7 @@ func TestCompatUnmarshal(t *testing.T) { "two-objs": {In: `{} {}`, ExpOut: nil, ExpErr: `invalid character '{' after top-level value`}, "two-numbers1": {In: `00`, ExpOut: nil, ExpErr: `invalid character '0' after top-level value`}, "two-numbers2": {In: `1 2`, ExpOut: nil, ExpErr: `invalid character '2' after top-level value`}, + "invalid-utf8": {In: "\x85", ExpErr: `invalid character '\u0085' looking for beginning of value`}, // 2e308 is slightly more than math.MaxFloat64 (~1.79e308) "obj-overflow": {In: `{"foo":"bar", "baz":2e308, "qux": "orb"}`, ExpOut: map[string]any{"foo": "bar", "baz": nil, "qux": "orb"}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, "ary-overflow": {In: `["foo",2e308,"bar",3e308]`, ExpOut: []any{"foo", nil, "bar", nil}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, @@ -223,6 +226,7 @@ func TestCompatDecode(t *testing.T) { "two-objs": {In: `{} {}`, ExpOut: map[string]any{}}, "two-numbers1": {In: `00`, ExpOut: float64(0)}, "two-numbers2": {In: `1 2`, ExpOut: float64(1)}, + "invalid-utf8": {In: "\x85", ExpErr: `invalid character '\u0085' looking for beginning of value`}, // 2e308 is slightly more than math.MaxFloat64 (~1.79e308) "obj-overflow": {In: `{"foo":"bar", "baz":2e308, "qux": "orb"}`, ExpOut: map[string]any{"foo": "bar", "baz": nil, "qux": "orb"}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, "ary-overflow": {In: `["foo",2e308,"bar",3e308]`, ExpOut: []any{"foo", nil, "bar", nil}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, diff --git a/compat/json/testdata/fuzz/FuzzEquiv/9e35149f0eb0866b b/compat/json/testdata/fuzz/FuzzEquiv/9e35149f0eb0866b new file mode 100644 index 0000000..bb8752b --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/9e35149f0eb0866b @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\x85") -- cgit v1.2.3-2-g168b From e38edfa53173c054ff97a5c51f90df0da60f16f5 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Feb 2023 18:55:57 -0700 Subject: jsonparse: Reword error messages to match encoding/json --- compat/json/compat_test.go | 18 ++++++++++++++++++ compat/json/testdata/fuzz/FuzzEquiv/1071d2f6e5b5f7d3 | 2 ++ compat/json/testdata/fuzz/FuzzEquiv/6bced2300496f15c | 2 ++ compat/json/testdata/fuzz/FuzzEquiv/6daf246742074967 | 2 ++ compat/json/testdata/fuzz/FuzzEquiv/7c3168c77fc059cb | 2 ++ compat/json/testdata/fuzz/FuzzEquiv/a955c588d78b5c3a | 2 ++ compat/json/testdata/fuzz/FuzzEquiv/cf667c6f1f3282c1 | 2 ++ compat/json/testdata/fuzz/FuzzEquiv/ef2c8755a89034da | 2 ++ 8 files changed, 32 insertions(+) create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/1071d2f6e5b5f7d3 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/6bced2300496f15c create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/6daf246742074967 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/7c3168c77fc059cb create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/a955c588d78b5c3a create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/cf667c6f1f3282c1 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/ef2c8755a89034da (limited to 'compat') diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 6aab103..cf9e359 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -188,6 +188,24 @@ func TestCompatUnmarshal(t *testing.T) { "obj-overflow": {In: `{"foo":"bar", "baz":2e308, "qux": "orb"}`, ExpOut: map[string]any{"foo": "bar", "baz": nil, "qux": "orb"}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, "ary-overflow": {In: `["foo",2e308,"bar",3e308]`, ExpOut: []any{"foo", nil, "bar", nil}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, "existing-overflow": {In: `2e308`, InPtr: func() any { x := 4; return &x }(), ExpOut: 4, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type int`}, + // syntax error messages + "syntax-01": {In: `{}x`, ExpErr: `invalid character 'x' after top-level value`}, + "syntax-02": {In: `x`, ExpErr: `invalid character 'x' looking for beginning of value`}, + "syntax-03": {In: `{x`, ExpErr: `invalid character 'x' looking for beginning of object key string`}, + "syntax-04": {In: `{""x`, ExpErr: `invalid character 'x' after object key`}, + "syntax-05": {In: `{"":0x`, ExpErr: `invalid character 'x' after object key:value pair`}, + "syntax-06": {In: `[0x`, ExpErr: `invalid character 'x' after array element`}, + "syntax-07": {In: "\"\x01\"", ExpErr: `invalid character '\x01' in string literal`}, + "syntax-08": {In: `"\x`, ExpErr: `invalid character 'x' in string escape code`}, + "syntax-09": {In: `"\ux`, ExpErr: `invalid character 'x' in \u hexadecimal character escape`}, + "syntax-10": {In: `"\u0x`, ExpErr: `invalid character 'x' in \u hexadecimal character escape`}, + "syntax-11": {In: `"\u00x`, ExpErr: `invalid character 'x' in \u hexadecimal character escape`}, + "syntax-12": {In: `"\u000x`, ExpErr: `invalid character 'x' in \u hexadecimal character escape`}, + "syntax-13": {In: `-x`, ExpErr: `invalid character 'x' in numeric literal`}, + "syntax-14": {In: `0.x`, ExpErr: `invalid character 'x' after decimal point in numeric literal`}, + "syntax-15": {In: `1ex`, ExpErr: `invalid character 'x' in exponent of numeric literal`}, + "syntax-16": {In: `1e+x`, ExpErr: `invalid character 'x' in exponent of numeric literal`}, + "syntax-17": {In: `fx`, ExpErr: `invalid character 'x' in literal false (expecting 'a')`}, } for tcName, tc := range testcases { tc := tc diff --git a/compat/json/testdata/fuzz/FuzzEquiv/1071d2f6e5b5f7d3 b/compat/json/testdata/fuzz/FuzzEquiv/1071d2f6e5b5f7d3 new file mode 100644 index 0000000..1095817 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/1071d2f6e5b5f7d3 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0EA") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/6bced2300496f15c b/compat/json/testdata/fuzz/FuzzEquiv/6bced2300496f15c new file mode 100644 index 0000000..4bc9c61 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/6bced2300496f15c @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("{0") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/6daf246742074967 b/compat/json/testdata/fuzz/FuzzEquiv/6daf246742074967 new file mode 100644 index 0000000..b1c3453 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/6daf246742074967 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\uX") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/7c3168c77fc059cb b/compat/json/testdata/fuzz/FuzzEquiv/7c3168c77fc059cb new file mode 100644 index 0000000..b95f079 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/7c3168c77fc059cb @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\x1e") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/a955c588d78b5c3a b/compat/json/testdata/fuzz/FuzzEquiv/a955c588d78b5c3a new file mode 100644 index 0000000..b135daa --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/a955c588d78b5c3a @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0.A") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/cf667c6f1f3282c1 b/compat/json/testdata/fuzz/FuzzEquiv/cf667c6f1f3282c1 new file mode 100644 index 0000000..f6ab571 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/cf667c6f1f3282c1 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\0") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/ef2c8755a89034da b/compat/json/testdata/fuzz/FuzzEquiv/ef2c8755a89034da new file mode 100644 index 0000000..7d9478d --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/ef2c8755a89034da @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0E+A") -- cgit v1.2.3-2-g168b From f68498a6fdb421483d9aebb45527452f6255bb68 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 25 Feb 2023 16:17:01 -0700 Subject: jsonparse: Don't show raw bytes as Unicode --- compat/json/compat_test.go | 8 ++++---- compat/json/equiv_test.go | 29 ++++++++++++++++++++++++++++- compat/json/testcompat_test.go | 2 +- 3 files changed, 33 insertions(+), 6 deletions(-) (limited to 'compat') diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index cf9e359..2380c53 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -79,7 +79,7 @@ func TestCompatCompact(t *testing.T) { "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, - "invalid-utf8": {In: "\x85", Err: `invalid character '\u0085' looking for beginning of value`}, + "invalid-utf8": {In: "\x85", Err: `invalid character '\x85' looking for beginning of value`}, } for tcName, tc := range testcases { tc := tc @@ -120,7 +120,7 @@ func TestCompatIndent(t *testing.T) { "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, - "invalid-utf8": {In: "\x85", Err: `invalid character '\u0085' looking for beginning of value`}, + "invalid-utf8": {In: "\x85", Err: `invalid character '\x85' looking for beginning of value`}, } for tcName, tc := range testcases { tc := tc @@ -183,7 +183,7 @@ func TestCompatUnmarshal(t *testing.T) { "two-objs": {In: `{} {}`, ExpOut: nil, ExpErr: `invalid character '{' after top-level value`}, "two-numbers1": {In: `00`, ExpOut: nil, ExpErr: `invalid character '0' after top-level value`}, "two-numbers2": {In: `1 2`, ExpOut: nil, ExpErr: `invalid character '2' after top-level value`}, - "invalid-utf8": {In: "\x85", ExpErr: `invalid character '\u0085' looking for beginning of value`}, + "invalid-utf8": {In: "\x85", ExpErr: `invalid character '\x85' looking for beginning of value`}, // 2e308 is slightly more than math.MaxFloat64 (~1.79e308) "obj-overflow": {In: `{"foo":"bar", "baz":2e308, "qux": "orb"}`, ExpOut: map[string]any{"foo": "bar", "baz": nil, "qux": "orb"}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, "ary-overflow": {In: `["foo",2e308,"bar",3e308]`, ExpOut: []any{"foo", nil, "bar", nil}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, @@ -244,7 +244,7 @@ func TestCompatDecode(t *testing.T) { "two-objs": {In: `{} {}`, ExpOut: map[string]any{}}, "two-numbers1": {In: `00`, ExpOut: float64(0)}, "two-numbers2": {In: `1 2`, ExpOut: float64(1)}, - "invalid-utf8": {In: "\x85", ExpErr: `invalid character '\u0085' looking for beginning of value`}, + "invalid-utf8": {In: "\x85", ExpErr: `invalid character '\x85' looking for beginning of value`}, // 2e308 is slightly more than math.MaxFloat64 (~1.79e308) "obj-overflow": {In: `{"foo":"bar", "baz":2e308, "qux": "orb"}`, ExpOut: map[string]any{"foo": "bar", "baz": nil, "qux": "orb"}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, "ary-overflow": {In: `["foo",2e308,"bar",3e308]`, ExpOut: []any{"foo", nil, "bar", nil}, ExpErr: `json: cannot unmarshal number 2e308 into Go value of type float64`}, diff --git a/compat/json/equiv_test.go b/compat/json/equiv_test.go index 246e4b3..cb02f43 100644 --- a/compat/json/equiv_test.go +++ b/compat/json/equiv_test.go @@ -44,8 +44,27 @@ func assertEquivErr(t *testing.T, stdErr, lowErr error) { lowByte := lowMsg[len(prefix)] if lowByte == '\\' { switch lowMsg[len(prefix)+1] { + case 'a': + lowByte = '\a' + case 'b': + lowByte = '\b' + case 'f': + lowByte = '\f' + case 'n': + lowByte = '\n' + case 'r': + lowByte = '\r' + case 't': + lowByte = '\t' + case 'v': + lowByte = '\v' + case '\\', '\'': + lowByte = lowMsg[len(prefix)+1] + case 'x': + lowByte64, _ := strconv.ParseUint(lowMsg[len(prefix)+2:][:2], 16, 8) + lowByte = byte(lowByte64) case 'u': - lowRune, _ := strconv.ParseUint(lowMsg[len(prefix)+2:][:4], 16, 32) + lowRune, _ := strconv.ParseUint(lowMsg[len(prefix)+2:][:4], 16, 16) var buf [4]byte utf8.EncodeRune(buf[:], rune(lowRune)) lowByte = buf[0] @@ -63,6 +82,14 @@ func assertEquivErr(t *testing.T, stdErr, lowErr error) { stdErr = errors.New(stdMsg) } } + + // I'd file a ticket for this, but @dsnet (one of the encoding/json maintainers) says that he's + // working on a parser-rewrite that would fix a bunch of this type of issue. + // https://github.com/golang/go/issues/58680#issuecomment-1444224084 + if strings.HasPrefix(stdMsg, `invalid character '\u00`) && strings.HasPrefix(lowMsg, `invalid character '\x`) { + stdMsg = `invalid character '\x` + strings.TrimPrefix(stdMsg, `invalid character '\u00`) + stdErr = errors.New(stdMsg) + } } // Text-equal. assert.Equal(t, stdErr.Error(), lowErr.Error()) diff --git a/compat/json/testcompat_test.go b/compat/json/testcompat_test.go index 73153d9..affcd7c 100644 --- a/compat/json/testcompat_test.go +++ b/compat/json/testcompat_test.go @@ -32,7 +32,7 @@ func checkValid(in []byte, scan *lowmemjson.ReEncoderConfig) error { func isValidNumber(s string) bool { var parser jsonparse.Parser for _, r := range s { - if t, _ := parser.HandleRune(r); !t.IsNumber() { + if t, _ := parser.HandleRune(r, true); !t.IsNumber() { return false } } -- cgit v1.2.3-2-g168b From 2a41777072f48467bef02bb3bd670d95c2b02102 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Feb 2023 18:55:57 -0700 Subject: compat/json: Handle io.EOF and io.ErrUnexpectedEOF the same --- compat/json/compat.go | 25 +++++++++++----------- compat/json/compat_test.go | 2 ++ .../json/testdata/fuzz/FuzzEquiv/930f49fab2367014 | 2 ++ .../json/testdata/fuzz/FuzzEquiv/caf81e9797b19c76 | 2 ++ 4 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/930f49fab2367014 create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/caf81e9797b19c76 (limited to 'compat') diff --git a/compat/json/compat.go b/compat/json/compat.go index 6f13fbb..4dc15ab 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -53,19 +53,18 @@ func convertError(err error, isUnmarshal bool) error { case *lowmemjson.DecodeReadError: return err case *lowmemjson.DecodeSyntaxError: - switch { - case errors.Is(err, io.EOF): - return io.EOF - case errors.Is(err, io.ErrUnexpectedEOF) && isUnmarshal: - return &SyntaxError{ - msg: "unexpected end of JSON input", - Offset: suberr.Offset, - } - default: - return &SyntaxError{ - msg: suberr.Err.Error(), - Offset: suberr.Offset + 1, + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + if isUnmarshal { + return &SyntaxError{ + msg: "unexpected end of JSON input", + Offset: suberr.Offset, + } } + return suberr.Err + } + return &SyntaxError{ + msg: suberr.Err.Error(), + Offset: suberr.Offset + 1, } case *lowmemjson.DecodeTypeError: switch subsuberr := suberr.Err.(type) { @@ -120,7 +119,7 @@ func convertError(err error, isUnmarshal bool) error { msg: err.Err.Error(), Offset: err.Offset + 1, } - if errors.Is(err, io.ErrUnexpectedEOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { ret.msg = "unexpected end of JSON input" } return ret diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 2380c53..43f17f1 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -72,6 +72,7 @@ func TestCompatCompact(t *testing.T) { Err string } testcases := map[string]testcase{ + "empty": {In: ``, Out: ``, Err: `unexpected end of JSON input`}, "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, "object": {In: `{}`, Out: `{}`}, "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, @@ -106,6 +107,7 @@ func TestCompatIndent(t *testing.T) { Err string } testcases := map[string]testcase{ + "empty": {In: ``, Out: ``, Err: `unexpected end of JSON input`}, "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, "object": {In: `{}`, Out: `{}`}, "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, diff --git a/compat/json/testdata/fuzz/FuzzEquiv/930f49fab2367014 b/compat/json/testdata/fuzz/FuzzEquiv/930f49fab2367014 new file mode 100644 index 0000000..7390d06 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/930f49fab2367014 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte(" ") diff --git a/compat/json/testdata/fuzz/FuzzEquiv/caf81e9797b19c76 b/compat/json/testdata/fuzz/FuzzEquiv/caf81e9797b19c76 new file mode 100644 index 0000000..67322c7 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/caf81e9797b19c76 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("") -- cgit v1.2.3-2-g168b From 4233e5012ece6d5a7fee3b5a518c41d916e1cf52 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 25 Feb 2023 18:12:15 -0700 Subject: compat/json: compat_test.go: Change the package to "json_test" --- compat/json/compat_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'compat') diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 43f17f1..3de48f7 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-2.0-or-later -package json +package json_test import ( "bytes" @@ -11,6 +11,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "git.lukeshu.com/go/lowmemjson/compat/json" ) func TestCompatHTMLEscape(t *testing.T) { @@ -31,7 +33,7 @@ func TestCompatHTMLEscape(t *testing.T) { t.Parallel() t.Logf("in=%q", tc.In) var dst bytes.Buffer - HTMLEscape(&dst, []byte(tc.In)) + json.HTMLEscape(&dst, []byte(tc.In)) assert.Equal(t, tc.Out, dst.String()) }) } @@ -58,7 +60,7 @@ func TestCompatValid(t *testing.T) { t.Run(tcName, func(t *testing.T) { t.Parallel() t.Logf("in=%q", tc.In) - act := Valid([]byte(tc.In)) + act := json.Valid([]byte(tc.In)) assert.Equal(t, tc.Exp, act) }) } @@ -88,7 +90,7 @@ func TestCompatCompact(t *testing.T) { t.Parallel() t.Logf("in=%q", tc.In) var out bytes.Buffer - err := Compact(&out, []byte(tc.In)) + err := json.Compact(&out, []byte(tc.In)) assert.Equal(t, tc.Out, out.String()) if tc.Err == "" { assert.NoError(t, err) @@ -130,7 +132,7 @@ func TestCompatIndent(t *testing.T) { t.Parallel() t.Logf("in=%q", tc.In) var out bytes.Buffer - err := Indent(&out, []byte(tc.In), ">", ".") + err := json.Indent(&out, []byte(tc.In), ">", ".") assert.Equal(t, tc.Out, out.String()) if tc.Err == "" { assert.NoError(t, err) @@ -157,7 +159,7 @@ func TestCompatMarshal(t *testing.T) { tc := tc t.Run(tcName, func(t *testing.T) { t.Parallel() - out, err := Marshal(tc.In) + out, err := json.Marshal(tc.In) assert.Equal(t, tc.Out, string(out)) if tc.Err == "" { assert.NoError(t, err) @@ -218,7 +220,7 @@ func TestCompatUnmarshal(t *testing.T) { var out any ptr = &out } - err := Unmarshal([]byte(tc.In), ptr) + err := json.Unmarshal([]byte(tc.In), ptr) assert.Equal(t, tc.ExpOut, reflect.ValueOf(ptr).Elem().Interface()) if tc.ExpErr == "" { assert.NoError(t, err) @@ -261,7 +263,7 @@ func TestCompatDecode(t *testing.T) { var out any ptr = &out } - err := NewDecoder(strings.NewReader(tc.In)).Decode(ptr) + err := json.NewDecoder(strings.NewReader(tc.In)).Decode(ptr) assert.Equal(t, tc.ExpOut, reflect.ValueOf(ptr).Elem().Interface()) if tc.ExpErr == "" { assert.NoError(t, err) -- cgit v1.2.3-2-g168b From 22edcf6a68a057ed04368d5f78c8ba3ddfee8d57 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 25 Feb 2023 11:11:36 -0700 Subject: reencode: Improve the error messages for trailing partial-UTF-8 --- compat/json/testdata/fuzz/FuzzEquiv/95640f7d88708118 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 compat/json/testdata/fuzz/FuzzEquiv/95640f7d88708118 (limited to 'compat') diff --git a/compat/json/testdata/fuzz/FuzzEquiv/95640f7d88708118 b/compat/json/testdata/fuzz/FuzzEquiv/95640f7d88708118 new file mode 100644 index 0000000..77924f3 --- /dev/null +++ b/compat/json/testdata/fuzz/FuzzEquiv/95640f7d88708118 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xf0") -- cgit v1.2.3-2-g168b