decode: Fix DecodeTypeError offsets

6 files changed, 144 insertions, 14 deletions

diff --git a/ReleaseNotes.md b/ReleaseNotes.md
index f19ce68..a2365f0 100644
--- a/ReleaseNotes.md
+++ b/ReleaseNotes.md
@@ -10,6 +10,9 @@
 
    - Encoder: `*EncodeMethodError` is now also used when a method
      produces invalid JSON.
+   - Decoder: The offset in `*DecodeTypeError`s now correctly point
+     the start of the value, rather than somewhere in the middle of
+     it.
 
 # v0.2.0 (2023-01-26)
 
diff --git a/compat/json/compat.go b/compat/json/compat.go
index 0c9e800..48d708b 100644
--- a/compat/json/compat.go
+++ b/compat/json/compat.go
@@ -183,7 +183,7 @@ func convertDecodeError(err error) error {
 			default:
 				err = &SyntaxError{
 					msg:    terr.Err.Error(),
-					Offset: terr.Offset,
+					Offset: terr.Offset + 1,
 				}
 			}
 		case *lowmemjson.DecodeTypeError:
diff --git a/decode.go b/decode.go
index 3c51c4b..a7536f5 100644
--- a/decode.go
+++ b/decode.go
@@ -91,6 +91,7 @@ type Decoder struct {
 
 	// state
 	err         error
+	posStack    []int64
 	structStack []decodeStackItem
 }
 
@@ -150,6 +151,14 @@ func (dec *Decoder) More() bool {
 	return e == nil && t != internal.RuneTypeEOF
 }
 
+func (dec *Decoder) posStackPush() {
+	dec.posStack = append(dec.posStack, dec.InputOffset())
+}
+
+func (dec *Decoder) posStackPop() {
+	dec.posStack = dec.posStack[:len(dec.posStack)-1]
+}
+
 func (dec *Decoder) structStackPush(par reflect.Type, idx any) {
 	dec.structStack = append(dec.structStack, decodeStackItem{par, idx})
 }
@@ -266,7 +275,7 @@ func (dec *Decoder) panicType(jTyp string, gTyp reflect.Type, err error) {
 			GoType:   gTyp,
 			JSONType: jTyp,
 			Err:      err,
-			Offset:   dec.InputOffset(),
+			Offset:   dec.posStack[len(dec.posStack)-1],
 		},
 	})
 }
@@ -381,6 +390,8 @@ var kind2bits = map[reflect.Kind]int{
 }
 
 func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
+	dec.posStackPush()
+	defer dec.posStackPop()
 	typ := val.Type()
 	switch {
 	case val.CanAddr() && reflect.PointerTo(typ) == rawMessagePtrType:
@@ -388,17 +399,17 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 		var buf bytes.Buffer
 		dec.scan(&buf)
 		if err := val.Addr().Interface().(*json.RawMessage).UnmarshalJSON(buf.Bytes()); err != nil {
-			dec.panicType(t.JSONType(), typ, err)
+			dec.panicType(t.JSONType(), reflect.PointerTo(typ), err)
 		}
 	case val.CanAddr() && reflect.PointerTo(typ).Implements(decodableType):
 		t := dec.peekRuneType()
 		obj := val.Addr().Interface().(Decodable)
 		l := dec.limitingScanner()
 		if err := obj.DecodeJSON(l); err != nil {
-			dec.panicType(t.JSONType(), typ, err)
+			dec.panicType(t.JSONType(), reflect.PointerTo(typ), err)
 		}
 		if _, _, err := l.ReadRune(); err != io.EOF {
-			dec.panicType(t.JSONType(), typ, fmt.Errorf("did not consume entire %s", t.JSONType()))
+			dec.panicType(t.JSONType(), reflect.PointerTo(typ), fmt.Errorf("did not consume entire %s", t.JSONType()))
 		}
 	case val.CanAddr() && reflect.PointerTo(typ).Implements(jsonUnmarshalerType):
 		t := dec.peekRuneType()
@@ -406,7 +417,7 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 		dec.scan(&buf)
 		obj := val.Addr().Interface().(json.Unmarshaler)
 		if err := obj.UnmarshalJSON(buf.Bytes()); err != nil {
-			dec.panicType(t.JSONType(), typ, err)
+			dec.panicType(t.JSONType(), reflect.PointerTo(typ), err)
 		}
 	case val.CanAddr() && reflect.PointerTo(typ).Implements(textUnmarshalerType):
 		if nullOK && dec.peekRuneType() == internal.RuneTypeNullN {
@@ -530,9 +541,13 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 			index := indexStruct(typ)
 			var nameBuf strings.Builder
 			dec.decodeObject(typ, func() {
+				dec.posStackPush()
+				defer dec.posStackPop()
 				nameBuf.Reset()
 				dec.decodeString(nil, &nameBuf)
 			}, func() {
+				dec.posStackPush()
+				defer dec.posStackPop()
 				name := nameBuf.String()
 				dec.structStackPush(typ, name)
 				defer dec.structStackPop()
@@ -613,17 +628,19 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 					val.Set(reflect.MakeMap(typ))
 				}
 				var nameBuf bytes.Buffer
+				var nameValPtr reflect.Value
 				dec.decodeObject(typ, func() {
+					dec.posStackPush()
+					defer dec.posStackPop()
 					nameBuf.Reset()
 					dec.decodeString(nil, &nameBuf)
-				}, func() {
 					nameValTyp := typ.Key()
-					nameValPtr := reflect.New(nameValTyp)
+					nameValPtr = reflect.New(nameValTyp)
 					switch {
 					case reflect.PointerTo(nameValTyp).Implements(textUnmarshalerType):
 						obj := nameValPtr.Interface().(encoding.TextUnmarshaler)
 						if err := obj.UnmarshalText(nameBuf.Bytes()); err != nil {
-							dec.panicType("string", nameValTyp, err)
+							dec.panicType("string", reflect.PointerTo(nameValTyp), err)
 						}
 					default:
 						switch nameValTyp.Kind() {
@@ -645,6 +662,9 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 							dec.panicType("object", typ, &DecodeArgumentError{Type: nameValTyp})
 						}
 					}
+				}, func() {
+					dec.posStackPush()
+					defer dec.posStackPop()
 					dec.structStackPush(typ, nameValPtr.Elem())
 					defer dec.structStackPop()
 
@@ -699,6 +719,8 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 					}
 					i := 0
 					dec.decodeArray(typ, func() {
+						dec.posStackPush()
+						defer dec.posStackPop()
 						dec.structStackPush(typ, i)
 						defer dec.structStackPop()
 						mValPtr := reflect.New(typ.Elem())
@@ -718,6 +740,8 @@ func (dec *Decoder) decode(val reflect.Value, nullOK bool) {
 			i := 0
 			n := val.Len()
 			dec.decodeArray(typ, func() {
+				dec.posStackPush()
+				defer dec.posStackPop()
 				dec.structStackPush(typ, i)
 				defer dec.structStackPop()
 				if i < n {
@@ -776,9 +800,13 @@ func (dec *Decoder) decodeAny() any {
 		typ := reflect.TypeOf(ret)
 		var nameBuf strings.Builder
 		dec.decodeObject(typ, func() {
+			dec.posStackPush()
+			defer dec.posStackPop()
 			nameBuf.Reset()
 			dec.decodeString(nil, &nameBuf)
 		}, func() {
+			dec.posStackPush()
+			defer dec.posStackPop()
 			name := nameBuf.String()
 			dec.structStackPush(typ, name)
 			defer dec.structStackPop()
@@ -789,6 +817,8 @@ func (dec *Decoder) decodeAny() any {
 		ret := []any{}
 		typ := reflect.TypeOf(ret)
 		dec.decodeArray(typ, func() {
+			dec.posStackPush()
+			defer dec.posStackPop()
 			dec.structStackPush(typ, len(ret))
 			defer dec.structStackPop()
 			ret = append(ret, dec.decodeAny())
@@ -840,8 +870,12 @@ func DecodeObject(r io.RuneScanner, decodeKey, decodeVal func(io.RuneScanner) er
 		}
 	}()
 	dec := NewDecoder(r)
+	dec.posStackPush()
+	defer dec.posStackPop()
 	dec.decodeObject(nil,
 		func() {
+			dec.posStackPush()
+			defer dec.posStackPop()
 			l := dec.limitingScanner()
 			if err := decodeKey(l); err != nil {
 				dec.panicType("string", nil, err)
@@ -851,6 +885,8 @@ func DecodeObject(r io.RuneScanner, decodeKey, decodeVal func(io.RuneScanner) er
 			}
 		},
 		func() {
+			dec.posStackPush()
+			defer dec.posStackPop()
 			t := dec.peekRuneType()
 			l := dec.limitingScanner()
 			if err := decodeVal(l); err != nil {
@@ -910,7 +946,11 @@ func DecodeArray(r io.RuneScanner, decodeMember func(r io.RuneScanner) error) (e
 		}
 	}()
 	dec := NewDecoder(r)
+	dec.posStackPush()
+	defer dec.posStackPop()
 	dec.decodeArray(nil, func() {
+		dec.posStackPush()
+		defer dec.posStackPop()
 		t := dec.peekRuneType()
 		l := dec.limitingScanner()
 		if err := decodeMember(l); err != nil {
diff --git a/decode_scan.go b/decode_scan.go
index 249975d..b9a5ea8 100644
--- a/decode_scan.go
+++ b/decode_scan.go
@@ -62,7 +62,7 @@ func (sc *runeTypeScannerImpl) Reset() {
 		sc.rType, err = sc.parser.HandleRune(sc.rRune)
 		if err != nil {
 			sc.rErr = &DecodeSyntaxError{
-				Offset: sc.offset,
+				Offset: sc.offset - int64(sc.rSize),
 				Err:    err,
 			}
 		} else {
@@ -89,7 +89,7 @@ func (sc *runeTypeScannerImpl) ReadRuneType() (rune, int, internal.RuneType, err
 			sc.rType, err = sc.parser.HandleRune(sc.rRune)
 			if err != nil {
 				sc.rErr = &DecodeSyntaxError{
-					Offset: sc.offset,
+					Offset: sc.offset - int64(sc.rSize),
 					Err:    err,
 				}
 			} else {
diff --git a/decode_scan_test.go b/decode_scan_test.go
index f5ceee0..6a430ab 100644
--- a/decode_scan_test.go
+++ b/decode_scan_test.go
@@ -141,9 +141,9 @@ func TestRuneTypeScanner(t *testing.T) {
 			{'[', 1, internal.RuneTypeArrayBeg, nil},
 			{'0', 1, internal.RuneTypeNumberIntZero, nil},
 			{',', 1, internal.RuneTypeArrayComma, nil},
-			{']', 1, internal.RuneTypeError, &DecodeSyntaxError{Offset: 5, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}},
-			{']', 1, internal.RuneTypeError, &DecodeSyntaxError{Offset: 5, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}},
-			{']', 1, internal.RuneTypeError, &DecodeSyntaxError{Offset: 5, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}},
+			{']', 1, internal.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}},
+			{']', 1, internal.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}},
+			{']', 1, internal.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}},
 		}},
 		"multi-value": {`1{}`, `}`, []ReadRuneTypeResult{
 			{'1', 1, internal.RuneTypeNumberIntDig, nil},
diff --git a/methods_test.go b/methods_test.go
index 46e2601..f5d5a9a 100644
--- a/methods_test.go
+++ b/methods_test.go
@@ -232,3 +232,90 @@ func TestMethodsEncode(t *testing.T) {
 		}
 	})
 }
+
+type tstDecoder struct {
+	n   int
+	err string
+}
+
+func (d *tstDecoder) DecodeJSON(r io.RuneScanner) error {
+	for i := 0; i < d.n; i++ {
+		if _, _, err := r.ReadRune(); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+	}
+	if len(d.err) > 0 {
+		return errors.New(d.err)
+	}
+	return nil
+}
+
+type strUnmarshaler struct {
+	err string
+}
+
+func (u *strUnmarshaler) UnmarshalJSON([]byte) error {
+	if u.err == "" {
+		return nil
+	}
+	return errors.New(u.err)
+}
+
+type textUnmarshaler struct {
+	err string
+}
+
+func (u *textUnmarshaler) UnmarshalText([]byte) error {
+	if u.err == "" {
+		return nil
+	}
+	return errors.New(u.err)
+}
+
+type errTextUnmarshaler struct {
+	S string
+}
+
+func (u *errTextUnmarshaler) UnmarshalText(dat []byte) error {
+	u.S = string(dat)
+	return errors.New("eee")
+}
+
+func TestMethodsDecode(t *testing.T) {
+	t.Parallel()
+	type testcase struct {
+		In          string
+		Obj         any
+		ExpectedErr string
+	}
+	testcases := map[string]testcase{
+		"decode-basic":           {In: `{}`, Obj: &tstDecoder{n: 2}},
+		"decode-basic-eof":       {In: `{}`, Obj: &tstDecoder{n: 5}},
+		"decode-syntax-error":    {In: `{x}`, Obj: &tstDecoder{n: 5}, ExpectedErr: `json: v: syntax error at input byte 1: object: unexpected character: 'x'`},
+		"unmarshal-syntax-error": {In: `{x}`, Obj: &strUnmarshaler{}, ExpectedErr: `json: v: syntax error at input byte 1: object: unexpected character: 'x'`},
+		"decode-short":           {In: `{}`, Obj: &tstDecoder{n: 1}, ExpectedErr: `json: v: cannot decode JSON object at input byte 0 into Go *lowmemjson_test.tstDecoder: did not consume entire object`},
+		"decode-err":             {In: `{}`, Obj: &tstDecoder{err: "xxx"}, ExpectedErr: `json: v: cannot decode JSON object at input byte 0 into Go *lowmemjson_test.tstDecoder: xxx`},
+		"decode-err2":            {In: `{}`, Obj: &tstDecoder{n: 1, err: "yyy"}, ExpectedErr: `json: v: cannot decode JSON object at input byte 0 into Go *lowmemjson_test.tstDecoder: yyy`},
+		"unmarshal-err":          {In: `{}`, Obj: &strUnmarshaler{err: "zzz"}, ExpectedErr: `json: v: cannot decode JSON object at input byte 0 into Go *lowmemjson_test.strUnmarshaler: zzz`},
+		"unmarshaltext":          {In: `""`, Obj: &textUnmarshaler{}},
+		"unmarshaltext-nonstr":   {In: `{}`, Obj: &textUnmarshaler{}, ExpectedErr: `json: v: cannot decode JSON object at input byte 0 into Go *lowmemjson_test.textUnmarshaler`},
+		"unmarshaltext-err":      {In: `""`, Obj: &textUnmarshaler{err: "zzz"}, ExpectedErr: `json: v: cannot decode JSON string at input byte 0 into Go *lowmemjson_test.textUnmarshaler: zzz`},
+		"unmarshaltext-mapkey":   {In: `{"a":1}`, Obj: new(map[errTextUnmarshaler]int), ExpectedErr: `json: v: cannot decode JSON string at input byte 1 into Go *lowmemjson_test.errTextUnmarshaler: eee`},
+	}
+	for tcName, tc := range testcases {
+		tc := tc
+		t.Run(tcName, func(t *testing.T) {
+			t.Parallel()
+			obj := tc.Obj
+			err := lowmemjson.NewDecoder(strings.NewReader(tc.In)).Decode(&obj)
+			if tc.ExpectedErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, tc.ExpectedErr)
+			}
+		})
+	}
+}