From edfc7aa91b542978ce28eb109b99a257650b62b4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 18 Feb 2023 21:57:39 -0700 Subject: decode_scan, jsonparse: Rework the behavior to make a bit more sense - decode_scan: Don't have .Reset() re-play an erroring rune - decode_Scan: Have RuneTypeEOF always be zero-width - jsonparse: Don't replace syntax errors with RuneTypeEOF if there's no barrier --- decode_scan_test.go | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) (limited to 'decode_scan_test.go') diff --git a/decode_scan_test.go b/decode_scan_test.go index eaf2f37..17c40d5 100644 --- a/decode_scan_test.go +++ b/decode_scan_test.go @@ -112,8 +112,8 @@ func TestRuneTypeScanner(t *testing.T) { {',', 1, jsonparse.RuneTypeArrayComma, nil}, {0, pushReadBarrier, 0, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, - {']', 1, jsonparse.RuneTypeEOF, nil}, - {0, unreadRune, 0, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, unreadRune, 0, ErrInvalidUnreadRune}, {0, popReadBarrier, 0, nil}, {']', 1, jsonparse.RuneTypeArrayEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, @@ -149,17 +149,17 @@ func TestRuneTypeScanner(t *testing.T) { "multi-value1": {`1{}`, `{}`, []ReadRuneTypeResult{ {0, pushReadBarrier, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, - {'{', 1, jsonparse.RuneTypeEOF, nil}, - {'{', 1, jsonparse.RuneTypeEOF, nil}, - {'{', 1, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, }}, "multi-value2": {`1{}`, ``, []ReadRuneTypeResult{ {0, pushReadBarrier, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, - {'{', 1, jsonparse.RuneTypeEOF, nil}, - {'{', 1, jsonparse.RuneTypeEOF, nil}, - {'{', 1, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {0, reset, 0, nil}, {0, pushReadBarrier, 0, nil}, @@ -196,11 +196,11 @@ func TestRuneTypeScanner(t *testing.T) { {0, popReadBarrier, 0, nil}, // Test main. {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, - {',', 1, jsonparse.RuneTypeEOF, nil}, - {',', 1, jsonparse.RuneTypeEOF, nil}, - {',', 1, jsonparse.RuneTypeEOF, nil}, + {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, + {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, + {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, }}, - "child-fragment": {`[1,` + `1,`, ``, []ReadRuneTypeResult{ + "child-fragment": {`[1,` + `1,`, `,`, []ReadRuneTypeResult{ // Child prefix. {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, @@ -208,9 +208,9 @@ func TestRuneTypeScanner(t *testing.T) { {0, pushReadBarrier, 0, nil}, // Test main. {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, - {',', 1, jsonparse.RuneTypeEOF, nil}, - {',', 1, jsonparse.RuneTypeEOF, nil}, - {',', 1, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "elem": {` { "foo" : 12.0 } `, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, @@ -225,22 +225,23 @@ func TestRuneTypeScanner(t *testing.T) { {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, - {'}', 1, jsonparse.RuneTypeEOF, nil}, - {'}', 1, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, - "invalid-number": {`1.2.3`, `.3`, []ReadRuneTypeResult{ + "invalid-number": {`1.2.3`, ``, []ReadRuneTypeResult{ {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'2', 1, jsonparse.RuneTypeNumberFracDig, nil}, - {'.', 1, jsonparse.RuneTypeEOF, nil}, + {'.', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 3, Err: fmt.Errorf("invalid character %q after top-level value", '.')}}, {0, reset, 0, nil}, - {'.', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 3, Err: fmt.Errorf("invalid character %q looking for beginning of value", '.')}}, + {'3', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, }}, - "trailing-garbage": {" 42 x", `x`, []ReadRuneTypeResult{ + "trailing-garbage": {" 42 x", ``, []ReadRuneTypeResult{ {0, pushReadBarrier, 0, nil}, {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, unreadRune, 0, nil}, @@ -249,10 +250,10 @@ func TestRuneTypeScanner(t *testing.T) { {0, pushReadBarrier, 0, nil}, {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, - {'x', 1, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {0, popReadBarrier, 0, nil}, - {'x', 1, jsonparse.RuneTypeEOF, nil}, + {'x', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q after top-level value", 'x')}}, }}, "unread-reset": {`{}`, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, -- cgit v1.2.3-2-g168b From ab0d686b9bb43a02f8d74c5e881782ab4e94e30b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 18 Feb 2023 12:53:05 -0700 Subject: jsonparse: Have PushReadBarrier reject trailing whitespace --- decode_scan_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) (limited to 'decode_scan_test.go') diff --git a/decode_scan_test.go b/decode_scan_test.go index 17c40d5..ee532c2 100644 --- a/decode_scan_test.go +++ b/decode_scan_test.go @@ -122,6 +122,32 @@ func TestRuneTypeScanner(t *testing.T) { {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "tail-ws": {`{"foo": 12.0} `, ``, []ReadRuneTypeResult{ + // Disable auto-child. + {0, pushReadBarrier, 0, nil}, + {0, popReadBarrier, 0, nil}, + // Test main. + {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, + {'"', 1, jsonparse.RuneTypeStringBeg, nil}, + {'f', 1, jsonparse.RuneTypeStringChar, nil}, + {'o', 1, jsonparse.RuneTypeStringChar, nil}, + {'o', 1, jsonparse.RuneTypeStringChar, nil}, + {'"', 1, jsonparse.RuneTypeStringEnd, nil}, + {':', 1, jsonparse.RuneTypeObjectColon, nil}, + {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, + {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, + {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + }}, + "child-tail-ws": {`[1,` + `{"foo": 12.0} `, ` `, []ReadRuneTypeResult{ + // Child prefix. + {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, + {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {',', 1, jsonparse.RuneTypeArrayComma, nil}, + {0, pushReadBarrier, 0, nil}, + // Test main. {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, @@ -213,6 +239,36 @@ func TestRuneTypeScanner(t *testing.T) { {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "elem": {` { "foo" : 12.0 } `, ``, []ReadRuneTypeResult{ + // Disable auto-child. + {0, pushReadBarrier, 0, nil}, + {0, popReadBarrier, 0, nil}, + // Test main. + {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, + {'"', 1, jsonparse.RuneTypeStringBeg, nil}, + {'f', 1, jsonparse.RuneTypeStringChar, nil}, + {'o', 1, jsonparse.RuneTypeStringChar, nil}, + {'o', 1, jsonparse.RuneTypeStringChar, nil}, + {'"', 1, jsonparse.RuneTypeStringEnd, nil}, + {':', 1, jsonparse.RuneTypeObjectColon, nil}, + {0, pushReadBarrier, 0, nil}, + {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, + {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, popReadBarrier, 0, nil}, + {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + {0, 0, jsonparse.RuneTypeEOF, nil}, + }}, + "child-elem": {`[1,` + ` { "foo" : 12.0 } `, ` `, []ReadRuneTypeResult{ + // Child prefix. + {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, + {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, + {',', 1, jsonparse.RuneTypeArrayComma, nil}, + {0, pushReadBarrier, 0, nil}, + // Test main. {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, -- cgit v1.2.3-2-g168b