From eaaf7bc29d43b4470623c75e6e409a049b3083af Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Feb 2023 11:02:33 -0700 Subject: compat/json: Valid: Check for EOF --- compat/json/compat.go | 9 +++++++-- compat/json/compat_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 compat/json/compat_test.go (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index c96470d..300ab2f 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -175,8 +175,13 @@ func Valid(data []byte) bool { formatter := lowmemjson.NewReEncoder(io.Discard, lowmemjson.ReEncoderConfig{ Compact: true, }) - _, err := formatter.Write(data) - return err == nil + if _, err := formatter.Write(data); err != nil { + return false + } + if err := formatter.Close(); err != nil { + return false + } + return true } // Decode wrappers /////////////////////////////////////////////////// diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go new file mode 100644 index 0000000..5c8f3ee --- /dev/null +++ b/compat/json/compat_test.go @@ -0,0 +1,34 @@ +// Copyright (C) 2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package json + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCompatValid(t *testing.T) { + t.Parallel() + type testcase struct { + In string + Exp bool + } + testcases := map[string]testcase{ + "empty": {In: ``, Exp: false}, + "num": {In: `1`, Exp: true}, + "trunc": {In: `{`, Exp: false}, + "object": {In: `{}`, Exp: true}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + t.Logf("in=%q", tc.In) + act := Valid([]byte(tc.In)) + assert.Equal(t, tc.Exp, act) + }) + } +} -- cgit v1.2.3-2-g168b From 49319198500729fd65bd6d69071f45f2d7ae2aa7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Feb 2023 11:02:09 -0700 Subject: compat/json: Compact, Indent: Clear the output if there's an error --- compat/json/compat.go | 14 ++++++++++-- compat/json/compat_test.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index 300ab2f..1cdbf0b 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -157,18 +157,28 @@ func reencode(dst io.Writer, src []byte, cfg lowmemjson.ReEncoderConfig) error { } func Compact(dst *bytes.Buffer, src []byte) error { - return reencode(dst, src, lowmemjson.ReEncoderConfig{ + start := dst.Len() + err := reencode(dst, src, lowmemjson.ReEncoderConfig{ Compact: true, BackslashEscape: lowmemjson.EscapePreserve, }) + if err != nil { + dst.Truncate(start) + } + return err } func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { - return reencode(dst, src, lowmemjson.ReEncoderConfig{ + start := dst.Len() + err := reencode(dst, src, lowmemjson.ReEncoderConfig{ Indent: indent, Prefix: prefix, BackslashEscape: lowmemjson.EscapePreserve, }) + if err != nil { + dst.Truncate(start) + } + return err } func Valid(data []byte) bool { diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 5c8f3ee..d513c27 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -5,6 +5,7 @@ package json import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -32,3 +33,59 @@ func TestCompatValid(t *testing.T) { }) } } + +func TestCompatCompact(t *testing.T) { + t.Parallel() + type testcase struct { + In string + Out string + Err string + } + testcases := map[string]testcase{ + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + t.Logf("in=%q", tc.In) + var out bytes.Buffer + err := Compact(&out, []byte(tc.In)) + assert.Equal(t, tc.Out, out.String()) + if tc.Err == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.Err) + } + }) + } +} + +func TestCompatIndent(t *testing.T) { + t.Parallel() + type testcase struct { + In string + Out string + Err string + } + testcases := map[string]testcase{ + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + t.Logf("in=%q", tc.In) + var out bytes.Buffer + err := Indent(&out, []byte(tc.In), ">", ".") + assert.Equal(t, tc.Out, out.String()) + if tc.Err == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.Err) + } + }) + } +} -- cgit v1.2.3-2-g168b From dfc67cecbd95344d296c31b537fa3ae8aec8c292 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Feb 2023 22:36:25 -0700 Subject: encode, reencode: Fix handling of invalid UTF-8 --- compat/json/compat.go | 5 ++++- compat/json/compat_test.go | 45 ++++++++++++++++++++++++++++++++++-------- compat/json/testcompat_test.go | 5 +++-- 3 files changed, 44 insertions(+), 11 deletions(-) (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index 1cdbf0b..d326514 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -160,6 +160,7 @@ func Compact(dst *bytes.Buffer, src []byte) error { start := dst.Len() err := reencode(dst, src, lowmemjson.ReEncoderConfig{ Compact: true, + InvalidUTF8: lowmemjson.InvalidUTF8Preserve, BackslashEscape: lowmemjson.EscapePreserve, }) if err != nil { @@ -173,6 +174,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { err := reencode(dst, src, lowmemjson.ReEncoderConfig{ Indent: indent, Prefix: prefix, + InvalidUTF8: lowmemjson.InvalidUTF8Preserve, BackslashEscape: lowmemjson.EscapePreserve, }) if err != nil { @@ -183,7 +185,8 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { func Valid(data []byte) bool { formatter := lowmemjson.NewReEncoder(io.Discard, lowmemjson.ReEncoderConfig{ - Compact: true, + Compact: true, + InvalidUTF8: lowmemjson.InvalidUTF8Error, }) if _, err := formatter.Write(data); err != nil { return false diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index d513c27..d989a4d 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -18,10 +18,11 @@ func TestCompatValid(t *testing.T) { Exp bool } testcases := map[string]testcase{ - "empty": {In: ``, Exp: false}, - "num": {In: `1`, Exp: true}, - "trunc": {In: `{`, Exp: false}, - "object": {In: `{}`, Exp: true}, + "empty": {In: ``, Exp: false}, + "num": {In: `1`, Exp: true}, + "trunc": {In: `{`, Exp: false}, + "object": {In: `{}`, Exp: true}, + "non-utf8": {In: "\"\x85\xcd\"", Exp: false}, // https://github.com/golang/go/issues/58517 } for tcName, tc := range testcases { tc := tc @@ -42,8 +43,9 @@ func TestCompatCompact(t *testing.T) { Err string } testcases := map[string]testcase{ - "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, - "object": {In: `{}`, Out: `{}`}, + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, } for tcName, tc := range testcases { tc := tc @@ -70,8 +72,9 @@ func TestCompatIndent(t *testing.T) { Err string } testcases := map[string]testcase{ - "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, - "object": {In: `{}`, Out: `{}`}, + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, } for tcName, tc := range testcases { tc := tc @@ -89,3 +92,29 @@ func TestCompatIndent(t *testing.T) { }) } } + +func TestCompatMarshal(t *testing.T) { + t.Parallel() + type testcase struct { + In any + Out string + Err string + } + testcases := map[string]testcase{ + "non-utf8": {In: "\x85\xcd", Out: "\"\\ufffd\\ufffd\""}, + "urc": {In: "\ufffd", Out: "\"\ufffd\""}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + out, err := Marshal(tc.In) + assert.Equal(t, tc.Out, string(out)) + if tc.Err == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.Err) + } + }) + } +} diff --git a/compat/json/testcompat_test.go b/compat/json/testcompat_test.go index 42cbf5c..e89b4b4 100644 --- a/compat/json/testcompat_test.go +++ b/compat/json/testcompat_test.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/json" "io" + "reflect" _ "unsafe" "git.lukeshu.com/go/lowmemjson" @@ -59,13 +60,13 @@ type encodeState struct { } func (es *encodeState) string(str string, _ bool) { - if err := jsonstring.EncodeStringFromString(&es.Buffer, lowmemjson.EscapeDefault, str); err != nil { + if err := jsonstring.EncodeStringFromString(&es.Buffer, lowmemjson.EscapeDefault, 0, reflect.Value{}, str); err != nil { panic(err) } } func (es *encodeState) stringBytes(str []byte, _ bool) { - if err := jsonstring.EncodeStringFromBytes(&es.Buffer, lowmemjson.EscapeDefault, str); err != nil { + if err := jsonstring.EncodeStringFromBytes(&es.Buffer, lowmemjson.EscapeDefault, 0, reflect.Value{}, str); err != nil { panic(err) } } -- cgit v1.2.3-2-g168b From 1a5b0561f53441d8a259a5096281699b5af16a6c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 16 Feb 2023 16:53:53 -0700 Subject: reencode: Add CompactFloats --- compat/json/compat_test.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'compat/json') diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index d989a4d..128bd1b 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -46,6 +46,7 @@ func TestCompatCompact(t *testing.T) { "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, "object": {In: `{}`, Out: `{}`}, "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, + "float": {In: `1.200e003`, Out: `1.200e003`}, } for tcName, tc := range testcases { tc := tc @@ -75,6 +76,7 @@ func TestCompatIndent(t *testing.T) { "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, "object": {In: `{}`, Out: `{}`}, "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, + "float": {In: `1.200e003`, Out: `1.200e003`}, } for tcName, tc := range testcases { tc := tc @@ -103,6 +105,7 @@ func TestCompatMarshal(t *testing.T) { testcases := map[string]testcase{ "non-utf8": {In: "\x85\xcd", Out: "\"\\ufffd\\ufffd\""}, "urc": {In: "\ufffd", Out: "\"\ufffd\""}, + "float": {In: 1.2e3, Out: `1200`}, } for tcName, tc := range testcases { tc := tc -- cgit v1.2.3-2-g168b From 2eb60b8be25a4b0fe3f1c5d5ca302e7e68190bad Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 16 Feb 2023 17:20:41 -0700 Subject: compat/json: Don't do actual JSON parsing in HTMLEscape --- compat/json/compat.go | 21 ++++++++++++++++++++- compat/json/compat_test.go | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index d326514..edc6908 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -11,10 +11,13 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io" "strconv" + "unicode/utf8" "git.lukeshu.com/go/lowmemjson" + "git.lukeshu.com/go/lowmemjson/internal/jsonstring" ) //nolint:stylecheck // ST1021 False positive; these aren't comments on individual types. @@ -144,7 +147,23 @@ func convertReEncodeError(err error) error { } func HTMLEscape(dst *bytes.Buffer, src []byte) { - _, _ = lowmemjson.NewReEncoder(dst, lowmemjson.ReEncoderConfig{}).Write(src) + for n := 0; n < len(src); { + c, size := utf8.DecodeRune(src[n:]) + if c == utf8.RuneError && size == 1 { + dst.WriteByte(src[n]) + } else { + mode := lowmemjson.EscapeHTMLSafe(c, lowmemjson.BackslashEscapeNone) + switch mode { + case lowmemjson.BackslashEscapeNone: + dst.WriteRune(c) + case lowmemjson.BackslashEscapeUnicode: + _ = jsonstring.WriteStringUnicodeEscape(dst, c) + default: + panic(fmt.Errorf("lowmemjson.EscapeHTMLSafe returned an unexpected escape mode=%d", mode)) + } + } + n += size + } } func reencode(dst io.Writer, src []byte, cfg lowmemjson.ReEncoderConfig) error { diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 128bd1b..0c14a60 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -11,6 +11,27 @@ import ( "github.com/stretchr/testify/assert" ) +func TestCompatHTMLEscape(t *testing.T) { + t.Parallel() + type testcase struct { + In string + Out string + } + testcases := map[string]testcase{ + "invalid": {In: `x`, Out: `x`}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + t.Logf("in=%q", tc.In) + var dst bytes.Buffer + HTMLEscape(&dst, []byte(tc.In)) + assert.Equal(t, tc.Out, dst.String()) + }) + } +} + func TestCompatValid(t *testing.T) { t.Parallel() type testcase struct { -- cgit v1.2.3-2-g168b From a87d6cbbb51a19071c5c742ef3c91bbb90a727c6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 16 Feb 2023 17:32:17 -0700 Subject: compat/json: Indent: Preserve trailing whitespace --- compat/json/compat.go | 21 ++++++++++++++++++++- compat/json/compat_test.go | 7 +++++++ compat/json/testcompat_test.go | 9 --------- 3 files changed, 27 insertions(+), 10 deletions(-) (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index edc6908..d33f278 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -188,6 +188,15 @@ func Compact(dst *bytes.Buffer, src []byte) error { return err } +func isSpace(c byte) bool { + switch c { + case 0x0020, 0x000A, 0x000D, 0x0009: + return true + default: + return false + } +} + func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { start := dst.Len() err := reencode(dst, src, lowmemjson.ReEncoderConfig{ @@ -198,8 +207,18 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { }) if err != nil { dst.Truncate(start) + return err } - return err + + // Preserve trailing whitespace. + lastNonWS := len(src) - 1 + for ; lastNonWS >= 0 && isSpace(src[lastNonWS]); lastNonWS-- { + } + if _, err := dst.Write(src[lastNonWS+1:]); err != nil { + return err + } + + return nil } func Valid(data []byte) bool { diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 0c14a60..c83ca7e 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -98,6 +98,13 @@ func TestCompatIndent(t *testing.T) { "object": {In: `{}`, Out: `{}`}, "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, "float": {In: `1.200e003`, Out: `1.200e003`}, + "tailws0": {In: `0`, Out: `0`}, + "tailws1": {In: `0 `, Out: `0 `}, + "tailws2": {In: `0 `, Out: `0 `}, + "tailws3": {In: "0\n", Out: "0\n"}, + "headws1": {In: ` 0`, Out: `0`}, + "objws1": {In: `{"a" : 1}`, Out: "{\n>.\"a\": 1\n>}"}, + "objws2": {In: "{\"a\"\n:\n1}", Out: "{\n>.\"a\": 1\n>}"}, } for tcName, tc := range testcases { tc := tc diff --git a/compat/json/testcompat_test.go b/compat/json/testcompat_test.go index e89b4b4..73153d9 100644 --- a/compat/json/testcompat_test.go +++ b/compat/json/testcompat_test.go @@ -46,15 +46,6 @@ const ( startDetectingCyclesAfter = 1000 ) -func isSpace(c byte) bool { - switch c { - case 0x0020, 0x000A, 0x000D, 0x0009: - return true - default: - return false - } -} - type encodeState struct { bytes.Buffer } -- cgit v1.2.3-2-g168b From 00187950437a10952b82353405e5ba4b4515fb29 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 16 Feb 2023 19:06:46 -0700 Subject: reencode: Don't normalize the capitalization of \uXXXX hex escapes --- compat/json/compat.go | 2 +- compat/json/compat_test.go | 54 ++++++++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 22 deletions(-) (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index d33f278..3a9bd6c 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -157,7 +157,7 @@ func HTMLEscape(dst *bytes.Buffer, src []byte) { case lowmemjson.BackslashEscapeNone: dst.WriteRune(c) case lowmemjson.BackslashEscapeUnicode: - _ = jsonstring.WriteStringUnicodeEscape(dst, c) + _ = jsonstring.WriteStringUnicodeEscape(dst, c, mode) default: panic(fmt.Errorf("lowmemjson.EscapeHTMLSafe returned an unexpected escape mode=%d", mode)) } diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index c83ca7e..29a8b37 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -18,7 +18,10 @@ func TestCompatHTMLEscape(t *testing.T) { Out string } testcases := map[string]testcase{ - "invalid": {In: `x`, Out: `x`}, + "invalid": {In: `x`, Out: `x`}, + "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, + "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, + "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, } for tcName, tc := range testcases { tc := tc @@ -39,11 +42,14 @@ func TestCompatValid(t *testing.T) { Exp bool } testcases := map[string]testcase{ - "empty": {In: ``, Exp: false}, - "num": {In: `1`, Exp: true}, - "trunc": {In: `{`, Exp: false}, - "object": {In: `{}`, Exp: true}, - "non-utf8": {In: "\"\x85\xcd\"", Exp: false}, // https://github.com/golang/go/issues/58517 + "empty": {In: ``, Exp: false}, + "num": {In: `1`, Exp: true}, + "trunc": {In: `{`, Exp: false}, + "object": {In: `{}`, Exp: true}, + "non-utf8": {In: "\"\x85\xcd\"", Exp: false}, // https://github.com/golang/go/issues/58517 + "hex-lower": {In: `"\uabcd"`, Exp: true}, + "hex-upper": {In: `"\uABCD"`, Exp: true}, + "hex-mixed": {In: `"\uAbCd"`, Exp: true}, } for tcName, tc := range testcases { tc := tc @@ -64,10 +70,13 @@ func TestCompatCompact(t *testing.T) { Err string } testcases := map[string]testcase{ - "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, - "object": {In: `{}`, Out: `{}`}, - "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, - "float": {In: `1.200e003`, Out: `1.200e003`}, + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, + "float": {In: `1.200e003`, Out: `1.200e003`}, + "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, + "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, + "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, } for tcName, tc := range testcases { tc := tc @@ -94,17 +103,20 @@ func TestCompatIndent(t *testing.T) { Err string } testcases := map[string]testcase{ - "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, - "object": {In: `{}`, Out: `{}`}, - "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, - "float": {In: `1.200e003`, Out: `1.200e003`}, - "tailws0": {In: `0`, Out: `0`}, - "tailws1": {In: `0 `, Out: `0 `}, - "tailws2": {In: `0 `, Out: `0 `}, - "tailws3": {In: "0\n", Out: "0\n"}, - "headws1": {In: ` 0`, Out: `0`}, - "objws1": {In: `{"a" : 1}`, Out: "{\n>.\"a\": 1\n>}"}, - "objws2": {In: "{\"a\"\n:\n1}", Out: "{\n>.\"a\": 1\n>}"}, + "trunc": {In: `{`, Out: ``, Err: `unexpected end of JSON input`}, + "object": {In: `{}`, Out: `{}`}, + "non-utf8": {In: "\"\x85\xcd\"", Out: "\"\x85\xcd\""}, + "float": {In: `1.200e003`, Out: `1.200e003`}, + "tailws0": {In: `0`, Out: `0`}, + "tailws1": {In: `0 `, Out: `0 `}, + "tailws2": {In: `0 `, Out: `0 `}, + "tailws3": {In: "0\n", Out: "0\n"}, + "headws1": {In: ` 0`, Out: `0`}, + "objws1": {In: `{"a" : 1}`, Out: "{\n>.\"a\": 1\n>}"}, + "objws2": {In: "{\"a\"\n:\n1}", Out: "{\n>.\"a\": 1\n>}"}, + "hex-lower": {In: `"\uabcd"`, Out: `"\uabcd"`}, + "hex-upper": {In: `"\uABCD"`, Out: `"\uABCD"`}, + "hex-mixed": {In: `"\uAbCd"`, Out: `"\uAbCd"`}, } for tcName, tc := range testcases { tc := tc -- cgit v1.2.3-2-g168b From 49ee8be679add0bd3cf08a2669331b3be7a835f8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 17 Feb 2023 19:21:37 -0700 Subject: compat/json: Correctly handle syntax-error-in-decode --- compat/json/compat.go | 82 +++++++++++++++++++++++++++++++++++++++++----- compat/json/compat_test.go | 78 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 9 deletions(-) (limited to 'compat/json') diff --git a/compat/json/compat.go b/compat/json/compat.go index 3a9bd6c..695c1a8 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -237,14 +237,14 @@ func Valid(data []byte) bool { // Decode wrappers /////////////////////////////////////////////////// -func convertDecodeError(err error) error { +func convertDecodeError(err error, isUnmarshal bool) error { if derr, ok := err.(*lowmemjson.DecodeError); ok { switch terr := derr.Err.(type) { case *lowmemjson.DecodeSyntaxError: switch { case errors.Is(terr.Err, io.EOF): err = io.EOF - case errors.Is(terr.Err, io.ErrUnexpectedEOF): + case errors.Is(terr.Err, io.ErrUnexpectedEOF) && isUnmarshal: err = &SyntaxError{ msg: "unexpected end of JSON input", Offset: terr.Offset, @@ -284,13 +284,66 @@ func convertDecodeError(err error) error { return err } +type decodeValidator struct{} + +func (*decodeValidator) DecodeJSON(r io.RuneScanner) error { + for { + if _, _, err := r.ReadRune(); err != nil { + + if err == io.EOF { + return nil + } + return err + } + } +} + +var _ lowmemjson.Decodable = (*decodeValidator)(nil) + func Unmarshal(data []byte, ptr any) error { - return convertDecodeError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(ptr)) + if err := convertDecodeError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(&decodeValidator{}), true); err != nil { + return err + } + if err := convertDecodeError(lowmemjson.NewDecoder(bytes.NewReader(data)).DecodeThenEOF(ptr), true); err != nil { + return err + } + return nil +} + +type teeRuneScanner struct { + src io.RuneScanner + dst *bytes.Buffer + lastSize int +} + +func (tee *teeRuneScanner) ReadRune() (r rune, size int, err error) { + r, size, err = tee.src.ReadRune() + if err == nil { + if _, err := tee.dst.WriteRune(r); err != nil { + return 0, 0, err + } + } + + tee.lastSize = size + return +} + +func (tee *teeRuneScanner) UnreadRune() error { + if tee.lastSize == 0 { + return lowmemjson.ErrInvalidUnreadRune + } + _ = tee.src.UnreadRune() + tee.dst.Truncate(tee.dst.Len() - tee.lastSize) + tee.lastSize = 0 + return nil } type Decoder struct { + validatorBuf *bufio.Reader + validator *lowmemjson.Decoder + + decoderBuf bytes.Buffer *lowmemjson.Decoder - buf *bufio.Reader } func NewDecoder(r io.Reader) *Decoder { @@ -298,18 +351,29 @@ func NewDecoder(r io.Reader) *Decoder { if !ok { br = bufio.NewReader(r) } - return &Decoder{ - Decoder: lowmemjson.NewDecoder(br), - buf: br, + ret := &Decoder{ + validatorBuf: br, } + ret.validator = lowmemjson.NewDecoder(&teeRuneScanner{ + src: ret.validatorBuf, + dst: &ret.decoderBuf, + }) + ret.Decoder = lowmemjson.NewDecoder(&ret.decoderBuf) + return ret } func (dec *Decoder) Decode(ptr any) error { - return convertDecodeError(dec.Decoder.Decode(ptr)) + if err := convertDecodeError(dec.validator.Decode(&decodeValidator{}), false); err != nil { + return err + } + if err := convertDecodeError(dec.Decoder.Decode(ptr), false); err != nil { + return err + } + return nil } func (dec *Decoder) Buffered() io.Reader { - dat, _ := dec.buf.Peek(dec.buf.Buffered()) + dat, _ := dec.validatorBuf.Peek(dec.validatorBuf.Buffered()) return bytes.NewReader(dat) } diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go index 29a8b37..df9d387 100644 --- a/compat/json/compat_test.go +++ b/compat/json/compat_test.go @@ -6,6 +6,8 @@ package json import ( "bytes" + "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -161,3 +163,79 @@ func TestCompatMarshal(t *testing.T) { }) } } + +func TestCompatUnmarshal(t *testing.T) { + t.Parallel() + type testcase struct { + In string + InPtr any + ExpOut any + ExpErr string + } + testcases := map[string]testcase{ + "empty-obj": {In: `{}`, ExpOut: map[string]any{}}, + "partial-obj": {In: `{"foo":"bar",`, ExpOut: nil, ExpErr: `unexpected end of JSON input`}, + "existing-obj": {In: `{"baz":"quz"}`, InPtr: &map[string]string{"foo": "bar"}, ExpOut: map[string]string{"foo": "bar", "baz": "quz"}}, + "existing-obj-partial": {In: `{"baz":"quz"`, InPtr: &map[string]string{"foo": "bar"}, ExpOut: map[string]string{"foo": "bar"}, ExpErr: "unexpected end of JSON input"}, + "empty-ary": {In: `[]`, ExpOut: []any{}}, + "two-objs": {In: `{} {}`, ExpOut: nil, ExpErr: `invalid character '{' after top-level value`}, + "two-numbers1": {In: `00`, ExpOut: nil, ExpErr: `invalid character '0' after top-level value`}, + "two-numbers2": {In: `1 2`, ExpOut: nil, ExpErr: `invalid character '2' after top-level value`}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + ptr := tc.InPtr + if ptr == nil { + var out any + ptr = &out + } + err := Unmarshal([]byte(tc.In), ptr) + assert.Equal(t, tc.ExpOut, reflect.ValueOf(ptr).Elem().Interface()) + if tc.ExpErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.ExpErr) + } + }) + } +} + +func TestCompatDecode(t *testing.T) { + t.Parallel() + type testcase struct { + In string + InPtr any + ExpOut any + ExpErr string + } + testcases := map[string]testcase{ + "empty-obj": {In: `{}`, ExpOut: map[string]any{}}, + "partial-obj": {In: `{"foo":"bar",`, ExpOut: nil, ExpErr: `unexpected EOF`}, + "existing-obj": {In: `{"baz":"quz"}`, InPtr: &map[string]string{"foo": "bar"}, ExpOut: map[string]string{"foo": "bar", "baz": "quz"}}, + "existing-obj-partial": {In: `{"baz":"quz"`, InPtr: &map[string]string{"foo": "bar"}, ExpOut: map[string]string{"foo": "bar"}, ExpErr: "unexpected EOF"}, + "empty-ary": {In: `[]`, ExpOut: []any{}}, + "two-objs": {In: `{} {}`, ExpOut: map[string]any{}}, + "two-numbers1": {In: `00`, ExpOut: float64(0)}, + "two-numbers2": {In: `1 2`, ExpOut: float64(1)}, + } + for tcName, tc := range testcases { + tc := tc + t.Run(tcName, func(t *testing.T) { + t.Parallel() + ptr := tc.InPtr + if ptr == nil { + var out any + ptr = &out + } + err := NewDecoder(strings.NewReader(tc.In)).Decode(ptr) + assert.Equal(t, tc.ExpOut, reflect.ValueOf(ptr).Elem().Interface()) + if tc.ExpErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.ExpErr) + } + }) + } +} -- cgit v1.2.3-2-g168b