wip: Reduce test differences [ci-skip]

 - Handle UTF-16 surrogate pairs
 - Handle cycles in values
 - Handle cycles in types
 - Better errors
 - Handle case-folding of struct field names
 - Allow []byteTypeWithMethods
 - Fix struct field-order
 - Fix handling of interfaces storing pointers
 - Enforce a maximum decode depth
 - Validate struct tags

7 files changed, 82 insertions, 40 deletions

diff --git a/compat/json/borrowed_decode_test.go b/compat/json/borrowed_decode_test.go
index 306f85e..4b84718 100644
--- a/compat/json/borrowed_decode_test.go
+++ b/compat/json/borrowed_decode_test.go
@@ -455,8 +455,8 @@ var unmarshalTests = []unmarshalTest{
 	{in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}},
 	{in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}},
 	{in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true},
-	{in: `[2, 3`, err: &SyntaxError{Err: "unexpected end of JSON input", Offset: 5}},                                                       // MODIFIED
-	{in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{Err: "invalid character '}' in numeric literal", Offset: 9}}, // MODIFIED
+	{in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}},
+	{in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}},
 
 	// raw value errors
 	{in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}},
@@ -957,7 +957,7 @@ var unmarshalTests = []unmarshalTest{
 		in:  `invalid`,
 		ptr: new(Number),
 		err: &SyntaxError{
-			Err:    "invalid character 'i' looking for beginning of value", // MODIFIED
+			msg:    "invalid character 'i' looking for beginning of value",
 			Offset: 1,
 		},
 	},
@@ -1040,7 +1040,6 @@ func TestMarshalNumberZeroVal(t *testing.T) {
 }
 
 func TestMarshalEmbeds(t *testing.T) {
-	t.Skip() // TODO
 	top := &Top{
 		Level0: 1,
 		Embed0: Embed0{
@@ -1089,17 +1088,16 @@ func equalError(a, b error) bool {
 	if b == nil {
 		return a == nil
 	}
-	return true // a.Error() == b.Error() // MODIFIED
+	return a.Error() == b.Error()
 }
 
 func TestUnmarshal(t *testing.T) {
-	t.Skip() // TODO
 	for i, tt := range unmarshalTests {
 		scan := lowmemjson.ReEncoder{Out: io.Discard} // MODIFIED
 		in := []byte(tt.in)
-		if _, err := scan.Write(in); err != nil {
+		if err := checkValid(in, &scan); err != nil {
 			if !equalError(err, tt.err) {
-				t.Errorf("#%d: checkValid: %#v\n\n%s", i, err, tt.in)
+				t.Errorf("#%d: checkValid: %#v", i, err)
 				continue
 			}
 		}
@@ -1142,11 +1140,11 @@ func TestUnmarshal(t *testing.T) {
 			continue
 		}
 		if !reflect.DeepEqual(v.Elem().Interface(), tt.out) {
-			t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v\n\n%s", i, v.Elem().Interface(), tt.out, tt.in)
+			t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), tt.out)
 			data, _ := Marshal(v.Elem().Interface())
-			println(string(data))
+			t.Log(string(data)) // MODIFIED
 			data, _ = Marshal(tt.out)
-			println(string(data))
+			t.Log(string(data)) // MODIFIED
 			continue
 		}
 
@@ -1311,7 +1309,7 @@ func TestErrorMessageFromMisusedString(t *testing.T) {
 		var s WrongString
 		err := NewDecoder(r).Decode(&s)
 		got := fmt.Sprintf("%v", err)
-		if err == nil { // if got != tt.err { // MODIFIED
+		if got != tt.err {
 			t.Errorf("%d. got err = %q, want %q", n, got, tt.err)
 		}
 	}
@@ -1742,7 +1740,6 @@ var interfaceSetTests = []struct {
 }
 
 func TestInterfaceSet(t *testing.T) {
-	t.Skip() // TODO
 	for _, tt := range interfaceSetTests {
 		b := struct{ X any }{tt.pre}
 		blob := `{"X":` + tt.json + `}`
@@ -2015,7 +2012,7 @@ var decodeTypeErrorTests = []struct {
 func TestUnmarshalTypeError(t *testing.T) {
 	for _, item := range decodeTypeErrorTests {
 		err := Unmarshal([]byte(item.src), item.dest)
-		if err == nil { // if _, ok := err.(*UnmarshalTypeError); !ok { // MODIFIED
+		if _, ok := err.(*UnmarshalTypeError); !ok {
 			t.Errorf("expected type error for Unmarshal(%q, type %T): got %T",
 				item.src, item.dest, err)
 		}
@@ -2037,7 +2034,7 @@ func TestUnmarshalSyntax(t *testing.T) {
 	var x any
 	for _, src := range unmarshalSyntaxTests {
 		err := Unmarshal([]byte(src), &x)
-		if err == nil { // _, ok := err.(*SyntaxError); !ok { // MODIFIED
+		if _, ok := err.(*SyntaxError); !ok {
 			t.Errorf("expected syntax error for Unmarshal(%q): got %T", src, err)
 		}
 	}
@@ -2202,9 +2199,9 @@ func TestInvalidUnmarshalText(t *testing.T) {
 			t.Errorf("Unmarshal expecting error, got nil")
 			continue
 		}
-		// if got := err.Error(); got != tt.want { // MODIFIED
-		// 	t.Errorf("Unmarshal = %q; want %q", got, tt.want) // MODIFIED
-		// } // MODIFIED
+		if got := err.Error(); got != tt.want {
+			t.Errorf("Unmarshal = %q; want %q", got, tt.want)
+		}
 	}
 }
 
@@ -2243,7 +2240,6 @@ func TestInvalidStringOption(t *testing.T) {
 // (Issue 28145) If the embedded struct is given an explicit name and has
 // exported methods, don't cause a panic trying to get its value.
 func TestUnmarshalEmbeddedUnexported(t *testing.T) {
-	t.Skip() // TODO
 	type (
 		embed1 struct{ Q int }
 		embed2 struct{ Q int }
@@ -2365,7 +2361,6 @@ func TestUnmarshalEmbeddedUnexported(t *testing.T) {
 }
 
 func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) {
-	t.Skip() // TODO
 	tests := []struct {
 		in  string
 		err error
@@ -2417,7 +2412,6 @@ func TestUnmarshalPanic(t *testing.T) {
 // The decoder used to hang if decoding into an interface pointing to its own address.
 // See golang.org/issues/31740.
 func TestUnmarshalRecursivePointer(t *testing.T) {
-	t.Skip() // TODO
 	var v any
 	v = &v
 	data := []byte(`{"a": "b"}`)
@@ -2493,7 +2487,6 @@ func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) {
 }
 
 func TestUnmarshalMaxDepth(t *testing.T) {
-	t.Skip() // TODO
 	testcases := []struct {
 		name        string
 		data        string
diff --git a/compat/json/borrowed_encode_test.go b/compat/json/borrowed_encode_test.go
index bb7c9dc..11c2db4 100644
--- a/compat/json/borrowed_encode_test.go
+++ b/compat/json/borrowed_encode_test.go
@@ -226,11 +226,11 @@ var unsupportedValues = []any{
 	math.NaN(),
 	math.Inf(-1),
 	math.Inf(1),
-	//pointerCycle, // MODIFIED
-	//pointerCycleIndirect, // MODIFIED
-	//mapCycle, // MODIFIED
-	//sliceCycle, // MODIFIED
-	//recursiveSliceCycle, // MODIFIED
+	pointerCycle,
+	pointerCycleIndirect,
+	mapCycle,
+	sliceCycle,
+	recursiveSliceCycle,
 }
 
 func TestUnsupportedValues(t *testing.T) {
@@ -344,7 +344,6 @@ func (CText) MarshalText() ([]byte, error) {
 }
 
 func TestMarshalerEscaping(t *testing.T) {
-	t.Skip() // MODIFIED
 	var c C
 	want := `"\u003c\u0026\u003e"`
 	b, err := Marshal(c)
@@ -877,7 +876,6 @@ func (f textfloat) MarshalText() ([]byte, error) { return tenc(`TF:%0.2f`, f) }
 
 // Issue 13783
 func TestEncodeBytekind(t *testing.T) {
-	t.Skip() // TODO
 	testdata := []struct {
 		data any
 		want string
@@ -1139,7 +1137,6 @@ func TestMarshalRawMessageValue(t *testing.T) {
 			if err != nil {
 				t.Errorf("test %d, unexpected failure: %v", i, err)
 			} else {
-				t.Skip() // MODIFIED
 				t.Errorf("test %d, unexpected success", i)
 			}
 		}
@@ -1178,7 +1175,6 @@ func TestMarshalUncommonFieldNames(t *testing.T) {
 	}
 }
 
-/* // MODIFIED
 func TestMarshalerError(t *testing.T) {
 	s := "test variable"
 	st := reflect.TypeOf(s)
@@ -1205,4 +1201,3 @@ func TestMarshalerError(t *testing.T) {
 		}
 	}
 }
-*/ // MODIFIED
diff --git a/compat/json/borrowed_misc.go b/compat/json/borrowed_misc.go
new file mode 100644
index 0000000..30a3b0e
--- /dev/null
+++ b/compat/json/borrowed_misc.go
@@ -0,0 +1,14 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+// A SyntaxError is a description of a JSON syntax error.
+// Unmarshal will return a SyntaxError if the JSON can't be parsed.
+type SyntaxError struct {
+	msg    string // description of error
+	Offset int64  // error occurred after reading Offset bytes
+}
+
+func (e *SyntaxError) Error() string { return e.msg }
diff --git a/compat/json/borrowed_scanner_test.go b/compat/json/borrowed_scanner_test.go
index 4955405..3474b3e 100644
--- a/compat/json/borrowed_scanner_test.go
+++ b/compat/json/borrowed_scanner_test.go
@@ -65,7 +65,6 @@ var ex1i = `[
 ]`
 
 func TestCompact(t *testing.T) {
-	t.Skip() // TODO
 	var buf bytes.Buffer
 	for _, tt := range examples {
 		buf.Reset()
@@ -86,7 +85,6 @@ func TestCompact(t *testing.T) {
 }
 
 func TestCompactSeparators(t *testing.T) {
-	t.Skip() // TODO
 	// U+2028 and U+2029 should be escaped inside strings.
 	// They should not appear outside strings.
 	tests := []struct {
@@ -106,7 +104,6 @@ func TestCompactSeparators(t *testing.T) {
 }
 
 func TestIndent(t *testing.T) {
-	t.Skip() // TODO
 	var buf bytes.Buffer
 	for _, tt := range examples {
 		buf.Reset()
@@ -192,7 +189,6 @@ var indentErrorTests = []indentErrorTest{
 }
 
 func TestIndentErrors(t *testing.T) {
-	t.Skip() // TODO
 	for i, tt := range indentErrorTests {
 		slice := make([]uint8, 0)
 		buf := bytes.NewBuffer(slice)
diff --git a/compat/json/borrowed_tagkey_test.go b/compat/json/borrowed_tagkey_test.go
index 6a2d612..6330efd 100644
--- a/compat/json/borrowed_tagkey_test.go
+++ b/compat/json/borrowed_tagkey_test.go
@@ -96,7 +96,6 @@ var structTagObjectKeyTests = []struct {
 }
 
 func TestStructTagObjectKey(t *testing.T) {
-	t.Skip() // TODO
 	for _, tt := range structTagObjectKeyTests {
 		b, err := Marshal(tt.raw)
 		if err != nil {
diff --git a/compat/json/compat.go b/compat/json/compat.go
index 78a9d5f..b26914b 100644
--- a/compat/json/compat.go
+++ b/compat/json/compat.go
@@ -8,7 +8,9 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"io"
+	"strconv"
 
 	"git.lukeshu.com/go/lowmemjson"
 )
@@ -19,7 +21,7 @@ type (
 	RawMessage = json.RawMessage
 
 	// low-level decode errors
-	SyntaxError         = lowmemjson.SyntaxError
+	//SyntaxError         = lowmemjson.DecodeSyntaxError // expose a field
 	UnmarshalFieldError = json.UnmarshalFieldError
 	UnmarshalTypeError  = json.UnmarshalTypeError // lowmemjson.DecodeTypeError
 
@@ -28,7 +30,7 @@ type (
 
 	// marshal errors
 	InvalidUTF8Error      = json.InvalidUTF8Error
-	MarshalerError        = json.MarshalerError
+	MarshalerError        = lowmemjson.EncodeMethodError // expose a field
 	UnsupportedTypeError  = json.UnsupportedTypeError
 	UnsupportedValueError = json.UnsupportedValueError
 )
@@ -92,7 +94,7 @@ func Valid(data []byte) bool {
 }
 
 func Unmarshal(data []byte, ptr any) error {
-	return lowmemjson.Decode(bytes.NewReader(data), ptr)
+	return NewDecoder(bytes.NewReader(data)).Decode(ptr)
 }
 
 /////////////////////////////////////////////////////////////////////
@@ -113,6 +115,43 @@ func NewDecoder(r io.Reader) *Decoder {
 	}
 }
 
+func (dec *Decoder) Decode(ptr any) error {
+	err := dec.Decoder.Decode(ptr)
+	if derr, ok := err.(*lowmemjson.DecodeError); ok {
+		switch terr := derr.Err.(type) {
+		case *lowmemjson.DecodeSyntaxError:
+			err = &SyntaxError{
+				msg:    terr.Err.Error(),
+				Offset: terr.Offset,
+			}
+		case *lowmemjson.DecodeTypeError:
+			if typeErr, ok := terr.Err.(*json.UnmarshalTypeError); ok {
+				err = &UnmarshalTypeError{
+					Value:  typeErr.Value,
+					Type:   typeErr.Type,
+					Offset: typeErr.Offset,
+					Struct: derr.FieldParent,
+					Field:  derr.FieldName,
+				}
+			} else if _, isArgErr := terr.Err.(*lowmemjson.DecodeArgumentError); terr.Err != nil &&
+				!isArgErr &&
+				!errors.Is(terr.Err, strconv.ErrSyntax) &&
+				!errors.Is(terr.Err, strconv.ErrRange) {
+				err = terr.Err
+			} else {
+				err = &UnmarshalTypeError{
+					Value:  terr.JSONType,
+					Type:   terr.GoType,
+					Offset: terr.Offset,
+					Struct: derr.FieldParent,
+					Field:  derr.FieldName,
+				}
+			}
+		}
+	}
+	return err
+}
+
 func (dec *Decoder) Buffered() io.Reader {
 	dat, _ := dec.buf.Peek(dec.buf.Buffered())
 	return bytes.NewReader(dat)
diff --git a/compat/json/compat_test.go b/compat/json/compat_test.go
index 399ff02..5a34d22 100644
--- a/compat/json/compat_test.go
+++ b/compat/json/compat_test.go
@@ -7,11 +7,17 @@ package json
 import (
 	"bytes"
 
+	"git.lukeshu.com/go/lowmemjson"
 	"git.lukeshu.com/go/lowmemjson/internal"
 )
 
 var parseTag = internal.ParseTag
 
+func checkValid(in []byte, scan *lowmemjson.ReEncoder) error {
+	_, err := scan.Write(in)
+	return err
+}
+
 const (
 	startDetectingCyclesAfter = 1000
 )