// Copyright (C) 2022-2023 Luke Shumaker // // SPDX-License-Identifier: GPL-2.0-or-later package lowmemjson import ( "fmt" "io" "strings" "testing" "github.com/stretchr/testify/assert" "git.lukeshu.com/go/lowmemjson/internal/jsonparse" ) type ReadRuneTypeResult struct { r rune s int t jsonparse.RuneType e error } const ( unreadRune = -1 pushReadBarrier = -2 popReadBarrier = -3 reset = -4 ) func (r ReadRuneTypeResult) String() string { switch r.s { case unreadRune: return fmt.Sprintf("{%q, unreadRune, %#v, %v}", r.r, r.t, r.e) case pushReadBarrier: return fmt.Sprintf("{%q, pushReadBarrier, %#v, %v}", r.r, r.t, r.e) case popReadBarrier: return fmt.Sprintf("{%q, popReadBarrier, %#v, %v}", r.r, r.t, r.e) case reset: return fmt.Sprintf("{%q, reset, %#v, %v}", r.r, r.t, r.e) default: return fmt.Sprintf("{%q, %d, %#v, %v}", r.r, r.s, r.t, r.e) } } type runeTypeScannerTestcase struct { Input string ExpRemainder string Exp []ReadRuneTypeResult } func TestRuneTypeScanner(t *testing.T) { t.Parallel() testcases := map[string]runeTypeScannerTestcase{ "basic": {`{"foo": 12.0}`, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "unread": {`{"foo": 12.0}`, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, unreadRune, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "unread2": {`{"foo": 12.0}`, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, unreadRune, 0, nil}, {0, unreadRune, 0, ErrInvalidUnreadRune}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "unread-eof": {`[1,2]`, ``, []ReadRuneTypeResult{ {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {',', 1, jsonparse.RuneTypeArrayComma, nil}, {0, pushReadBarrier, 0, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, unreadRune, 0, ErrInvalidUnreadRune}, {0, popReadBarrier, 0, nil}, {']', 1, jsonparse.RuneTypeArrayEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, unreadRune, 0, ErrInvalidUnreadRune}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "tail-ws": {`{"foo": 12.0} `, ``, []ReadRuneTypeResult{ // Disable auto-child. {0, pushReadBarrier, 0, nil}, {0, popReadBarrier, 0, nil}, // Test main. {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "child-tail-ws": {`[1,` + `{"foo": 12.0} `, ` `, []ReadRuneTypeResult{ // Child prefix. {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {',', 1, jsonparse.RuneTypeArrayComma, nil}, {0, pushReadBarrier, 0, nil}, // Test main. {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "syntax-error": {`[[0,]`, ``, []ReadRuneTypeResult{ {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'0', 1, jsonparse.RuneTypeNumberIntZero, nil}, {',', 1, jsonparse.RuneTypeArrayComma, nil}, {']', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}}, {']', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}}, {']', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}}, }}, "multi-value1": {`1{}`, `{}`, []ReadRuneTypeResult{ {0, pushReadBarrier, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, }}, "multi-value2": {`1{}`, ``, []ReadRuneTypeResult{ {0, pushReadBarrier, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {0, reset, 0, nil}, {0, pushReadBarrier, 0, nil}, {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "early-eof": {` {`, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 2, Err: io.ErrUnexpectedEOF}}, {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 2, Err: io.ErrUnexpectedEOF}}, {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 2, Err: io.ErrUnexpectedEOF}}, }}, "empty": {``, ``, []ReadRuneTypeResult{ {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 0, Err: io.EOF}}, {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 0, Err: io.EOF}}, {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 0, Err: io.EOF}}, }}, "basic2": {`1`, ``, []ReadRuneTypeResult{ {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "fragment": {`1,`, ``, []ReadRuneTypeResult{ // Disable auto-child. {0, pushReadBarrier, 0, nil}, {0, popReadBarrier, 0, nil}, // Test main. {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, }}, "child-fragment": {`[1,` + `1,`, `,`, []ReadRuneTypeResult{ // Child prefix. {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {',', 1, jsonparse.RuneTypeArrayComma, nil}, {0, pushReadBarrier, 0, nil}, // Test main. {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "elem": {` { "foo" : 12.0 } `, ``, []ReadRuneTypeResult{ // Disable auto-child. {0, pushReadBarrier, 0, nil}, {0, popReadBarrier, 0, nil}, // Test main. {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {0, pushReadBarrier, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "child-elem": {`[1,` + ` { "foo" : 12.0 } `, ` `, []ReadRuneTypeResult{ // Child prefix. {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {',', 1, jsonparse.RuneTypeArrayComma, nil}, {0, pushReadBarrier, 0, nil}, // Test main. {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'"', 1, jsonparse.RuneTypeStringBeg, nil}, {'f', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'o', 1, jsonparse.RuneTypeStringChar, nil}, {'"', 1, jsonparse.RuneTypeStringEnd, nil}, {':', 1, jsonparse.RuneTypeObjectColon, nil}, {0, pushReadBarrier, 0, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "invalid-number": {`1.2.3`, ``, []ReadRuneTypeResult{ {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, {'2', 1, jsonparse.RuneTypeNumberFracDig, nil}, {'.', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 3, Err: fmt.Errorf("invalid character %q after top-level value", '.')}}, {0, reset, 0, nil}, {'3', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, "trailing-garbage": {" 42 x", ``, []ReadRuneTypeResult{ {0, pushReadBarrier, 0, nil}, {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, unreadRune, 0, nil}, {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, unreadRune, 0, nil}, {0, pushReadBarrier, 0, nil}, {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, popReadBarrier, 0, nil}, {0, popReadBarrier, 0, nil}, {'x', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q after top-level value", 'x')}}, }}, "unread-reset": {`{}`, ``, []ReadRuneTypeResult{ {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {0, unreadRune, 0, nil}, {0, reset, 0, nil}, {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, {0, 0, jsonparse.RuneTypeEOF, nil}, }}, } func() { childTestcases := make(map[string]runeTypeScannerTestcase) for tcName, tc := range testcases { canChild := true for _, res := range tc.Exp { if res.s == pushReadBarrier || res.s == reset { canChild = false break } } if !canChild { continue } tc.Input = `[1,` + tc.Input tc.Exp = append([]ReadRuneTypeResult{ {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, {',', 1, jsonparse.RuneTypeArrayComma, nil}, {0, pushReadBarrier, 0, nil}, }, tc.Exp...) for i := 2; i < len(tc.Exp); i++ { if se, ok := tc.Exp[i].e.(*DecodeSyntaxError); ok { seCopy := *se seCopy.Offset += 3 tc.Exp[i].e = &seCopy } } childTestcases["child-"+tcName] = tc } for tcName, tc := range childTestcases { testcases[tcName] = tc } }() for tcName, tc := range testcases { tc := tc t.Run(tcName, func(t *testing.T) { t.Parallel() t.Logf("input=%q", tc.Input) reader := strings.NewReader(tc.Input) sc := &runeTypeScanner{inner: reader} var exp, act []string for _, iExp := range tc.Exp { var iAct ReadRuneTypeResult switch iExp.s { case unreadRune: iAct.s = iExp.s iAct.e = sc.UnreadRune() case pushReadBarrier: sc.PushReadBarrier() iAct.s = iExp.s case popReadBarrier: sc.PopReadBarrier() iAct.s = iExp.s case reset: sc.Reset() iAct.s = iExp.s default: iAct.r, iAct.s, iAct.t, iAct.e = sc.ReadRuneType() } exp = append(exp, iExp.String()) act = append(act, iAct.String()) } assert.Equal(t, exp, act) assert.Equal(t, tc.ExpRemainder, tc.Input[len(tc.Input)-reader.Len():]) }) } }