// Copyright (C) 2022-2023 Luke Shumaker <lukeshu@lukeshu.com>
//
// SPDX-License-Identifier: GPL-2.0-or-later
//
// Some doc comments are
// copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// SPDX-License-Identifier: BSD-3-Clause
package lowmemjson
import (
"bytes"
"encoding"
"fmt"
"io"
"reflect"
"strconv"
"strings"
"unicode/utf16"
"unicode/utf8"
"git.lukeshu.com/go/lowmemjson/internal/base64dec"
"git.lukeshu.com/go/lowmemjson/internal/fastio"
"git.lukeshu.com/go/lowmemjson/internal/jsonparse"
"git.lukeshu.com/go/lowmemjson/internal/jsonstruct"
)
// Decodable is the interface implemented by types that can decode a
// JSON representation of themselves. Decodable is a
// low-memory-overhead replacement for the json.Unmarshaler interface.
//
// On the io.RuneScanner passed to DecodeJSON:
//
// - .UnreadRune() will return ErrInvalidUnreadRune if the last
// operation was not a successful .ReadRune() call.
//
// - .ReadRune() will return io.EOF at the end of the JSON value; it
// is not possible for .ReadRune() to read past the end of the
// value in to another value.
//
// - .ReadRune() will never return invalid JSON; if invalid JSON is
// encountered, it will use a panic-based mechanism to transfer
// control back to the Decoder.
//
// - .ReadRune() never return an error other than io.EOF; if an I/O
// error is encountered, it will use a panic-based mechanism to
// transfer control back to the Decoder.
//
// DecodeJSON is expected to consume the entire scanner until io.EOF
// or another is encountered; if it does not, then the parent Decode
// call will return a *DecodeTypeError.
//
// Implementor's note: "withLimitingScanner" is the thing to search
// for in decode.go if you want to read up on that io.RuneScanner.
type Decodable interface {
DecodeJSON(io.RuneScanner) error
}
type decodeStackItem struct {
par reflect.Type
idx any
}
// A Decoder reads and decodes values from an input stream of JSON
// elements.
//
// Decoder is analogous to, and has a similar API to the standard
// library's encoding/json.Decoder. Differences are:
//
// - lowmemjson.NewDecoder takes an io.RuneScanner, while
// json.NewDecoder takes an io.Reader.
//
// - lowmemjson.Decoder does not have a .Buffered() method, while
// json.Decoder does.
//
// - lowmemjson.Decoder does not have a .Token() method, while
// json.Decoder does.
//
// If something more similar to a json.Decoder is desired,
// lowmemjson/compat/json.NewDecoder takes an io.Reader (and turns it
// into an io.RuneScanner by wrapping it in a bufio.Reader), and
// lowmemjson/compat/json.Decoder has a .Buffered() method; though
// lowmemjson/compat/json.Decoder also lacks the .Token() method.
type Decoder struct {
io runeTypeScanner
// config
disallowUnknownFields bool
useNumber bool
// state
posStack []int64
structStack []decodeStackItem
}
const maxNestingDepth = 10000
// NewDecoder returns a new Decoder that reads from r.
//
// NewDecoder is analogous to the standard library's
// encoding/json.NewDecoder, but takes an io.RuneScanner rather than
// an io.Reader.
func NewDecoder(r io.RuneScanner) *Decoder {
return &Decoder{
io: runeTypeScanner{
inner: r,
parser: jsonparse.Parser{
MaxDepth: maxNestingDepth,
},
},
}
}
// DisallowUnknownFields causes the Decoder to return an error when
// the destination is a struct and the input contains object keys
// which do not match any non-ignored, exported fields in the
// destination.
//
// This is identical to the standard library's
// encoding/json.Decoder.DisallowUnknownFields.
func (dec *Decoder) DisallowUnknownFields() { dec.disallowUnknownFields = true }
// UseNumber causes the Decoder to unmarshal a number into an
// interface{} as a Number instead of as a float64.
//
// This is identical to the standard library's
// encoding/json.Decoder.UseNumber.
func (dec *Decoder) UseNumber() { dec.useNumber = true }
// InputOffset returns the input stream byte offset of the current
// decoder position. The offset gives the location of the rune that
// will be returned from the next call to .ReadRune().
//
// This is identical to the standard library's
// encoding/json.Decoder.InputOffset.
func (dec *Decoder) InputOffset() int64 { return dec.io.InputOffset() }
// More reports whether there is more to the stream of JSON elements,
// or if the Decoder has reached EOF or an error.
//
// More is identical to the standard library's
// encoding/json.Decoder.More.
func (dec *Decoder) More() bool {
dec.io.Reset()
_, _, t, e := dec.io.ReadRuneType()
_ = dec.io.UnreadRune() // best effort
return e == nil && t != jsonparse.RuneTypeEOF
}
func (dec *Decoder) posStackPush() {
dec.posStack = append(dec.posStack, dec.InputOffset())
}
func (dec *Decoder) posStackPop() {
dec.posStack = dec.posStack[:len(dec.posStack)-1]
}
func (dec *Decoder) structStackPush(par reflect.Type, idx any) {
dec.structStack = append(dec.structStack, decodeStackItem{par, idx})
}
func (dec *Decoder) structStackPop() {
dec.structStack = dec.structStack[:len(dec.structStack)-1]
}
func (dec *Decoder) structStackStr() string {
var buf strings.Builder
buf.WriteString("v")
for _, item := range dec.structStack {
fmt.Fprintf(&buf, "[%#v]", item.idx)
}
return buf.String()
}
func (dec *Decoder) structStackParent() string {
last := len(dec.structStack) - 1
if last > 0 && dec.structStack[last].par.Kind() != reflect.Struct && dec.structStack[last-1].par.Kind() == reflect.Struct {
last--
}
if last >= 0 && dec.structStack[last].par.Kind() == reflect.Struct {
return dec.structStack[last].par.Name()
}
return ""
}
func (dec *Decoder) structStackName() string {
if dec.structStackParent() == "" {
return ""
}
var fields []string
for _, elem := range dec.structStack {
if elem.par.Kind() == reflect.Struct {
fields = append(fields, elem.idx.(string))
}
}
return strings.Join(fields, ".")
}
// DecodeThenEOF is like Decode, but emits an error if there is extra
// data after the JSON. A JSON document is specified to be a single
// JSON element; repeated calls to Decoder.Decode will happily decode
// a stream of multiple JSON elements.
func (dec *Decoder) DecodeThenEOF(ptr any) (err error) {
if err := dec.Decode(ptr); err != nil {
return err
}
c, s, t, _ := dec.io.ReadRuneType()
if t != jsonparse.RuneTypeEOF {
panic(fmt.Errorf("should not happen: .ReadRuneType returned non-EOF after decode without .Reset being called: %v", t))
}
if s > 0 {
return &DecodeError{
Err: &DecodeSyntaxError{
Err: fmt.Errorf("invalid character %q after top-level value", c),
Offset: dec.InputOffset(),
},
}
}
return nil
}
// Decode reads the next JSON element from the Decoder's input stream
// and stores it in the value pointed to by ptr.
//
// See the [documentation for encoding/json.Unmarshal] for details
// about the conversion of JSON into a Go value; Decode behaves
// identically to that, with the exception that in addition to the
// json.Unmarshaler interface it also checks for the Decodable
// interface.
//
// [documentation for encoding/json.Unmarshal]: https://pkg.go.dev/encoding/json@go1.20#Unmarshal
func (dec *Decoder) Decode(ptr any) (err error) {
ptrVal := reflect.ValueOf(ptr)
if ptrVal.Kind() != reflect.Pointer || ptrVal.IsNil() || !ptrVal.Elem().CanSet() {
return &DecodeArgumentError{
// don't use ptrVal.Type() because ptrVal might be invalid if ptr==nil
Type: reflect.TypeOf(ptr),
}
}
dec.io.Reset()
dec.io.PushReadBarrier()
if err := dec.decode(ptrVal.Elem(), false); err != nil {
return err
}
dec.io.PopReadBarrier()
return nil
}
// io helpers //////////////////////////////////////////////////////////////////////////////////////
func (dec *Decoder) newTypeError(jTyp string, gTyp reflect.Type, err error) *DecodeError {
return &DecodeError{
Field: dec.structStackStr(),
FieldParent: dec.structStackParent(),
FieldName: dec.structStackName(),
Err: &DecodeTypeError{
GoType: gTyp,
JSONType: jTyp,
Err: err,
Offset: dec.posStack[len(dec.posStack)-1],
},
}
}
func (dec *Decoder) readRune() (rune, jsonparse.RuneType, *DecodeError) {
c, _, t, e := dec.io.ReadRuneType()
if e != nil {
return 0, 0, &DecodeError{
Field: dec.structStackStr(),
FieldParent: dec.structStackParent(),
FieldName: dec.structStackName(),
Err: e,
}
}
return c, t, nil
}
func (dec *Decoder) unreadRune() {
if err := dec.io.UnreadRune(); err != nil {
// .UnreadRune() must succeed if the previous call was
// .ReadRune(), which it always is for this code.
panic(fmt.Errorf("should not happen: UnreadRune: %w", err))
}
}
func (dec *Decoder) peekRuneType() (jsonparse.RuneType, *DecodeError) {
_, t, err := dec.readRune()
if err != nil {
return 0, err
}
dec.unreadRune()
return t, nil
}
func (dec *Decoder) expectRune(ec rune, et jsonparse.RuneType) *DecodeError {
ac, at, err := dec.readRune()
if err != nil {
return err
}
if ac != ec || at != et {
panic(fmt.Errorf("should not happen: expected %q/%v but got %q/%v", ec, et, ac, at))
}
return nil
}
func (dec *Decoder) expectRuneType(ec rune, et jsonparse.RuneType, gt reflect.Type) *DecodeError {
ac, at, err := dec.readRune()
if err != nil {
return err
}
if ac != ec || at != et {
return dec.newTypeError(at.JSONType(), gt, nil)
}
return nil
}
type decRuneScanner struct {
dec *Decoder
eof bool
}
type scannerPanic *DecodeError
func (sc *decRuneScanner) ReadRune() (rune, int, error) {
if sc.eof {
return 0, 0, io.EOF
}
c, s, t, e := sc.dec.io.ReadRuneType()
if t == jsonparse.RuneTypeEOF {
sc.eof = true
sc.dec.io.PopReadBarrier()
return 0, 0, io.EOF
}
if e != nil {
panic(scannerPanic(&DecodeError{
Field: sc.dec.structStackStr(),
FieldParent: sc.dec.structStackParent(),
FieldName: sc.dec.structStackName(),
Err: e,
}))
}
return c, s, nil
}
func (sc *decRuneScanner) UnreadRune() error {
return sc.dec.io.UnreadRune()
}
func (dec *Decoder) withLimitingScanner(fn func(io.RuneScanner) *DecodeError) (err *DecodeError) {
dec.io.PushReadBarrier()
defer func() {
if r := recover(); r != nil {
if sp, ok := r.(scannerPanic); ok {
err = (*DecodeError)(sp)
} else {
panic(r)
}
}
}()
if err := fn(&decRuneScanner{dec: dec}); err != nil {
return err
}
return nil
}
// decoder main ////////////////////////////////////////////////////////////////////////////////////
var kind2bits = map[reflect.Kind]int{
reflect.Int: int(32 << (^uint(0) >> 63)),
reflect.Int8: 8,
reflect.Int16: 16,
reflect.Int32: 32,
reflect.Int64: 64,
reflect.Uint: int(32 << (^uint(0) >> 63)),
reflect.Uint8: 8,
reflect.Uint16: 16,
reflect.Uint32: 32,
reflect.Uint64: 64,
reflect.Uintptr: int(32 << (^uintptr(0) >> 63)),
reflect.Float32: 32,
reflect.Float64: 64,
}
func (dec *Decoder) decode(val reflect.Value, nullOK bool) *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
typ := val.Type()
switch {
case val.CanAddr() && reflect.PointerTo(typ) == rawMessagePtrType:
t, err := dec.peekRuneType()
if err != nil {
return err
}
var buf bytes.Buffer
if err := dec.scan(&buf); err != nil {
return err
}
if err := val.Addr().Interface().(*RawMessage).UnmarshalJSON(buf.Bytes()); err != nil {
return dec.newTypeError(t.JSONType(), reflect.PointerTo(typ), err)
}
case val.CanAddr() && reflect.PointerTo(typ).Implements(decodableType):
t, err := dec.peekRuneType()
if err != nil {
return err
}
obj := val.Addr().Interface().(Decodable)
return dec.withLimitingScanner(func(l io.RuneScanner) *DecodeError {
if err := obj.DecodeJSON(l); err != nil {
return dec.newTypeError(t.JSONType(), reflect.PointerTo(typ), err)
}
if _, _, err := l.ReadRune(); err != io.EOF {
return dec.newTypeError(t.JSONType(), reflect.PointerTo(typ), fmt.Errorf("did not consume entire %s", t.JSONType()))
}
return nil
})
case val.CanAddr() && reflect.PointerTo(typ).Implements(jsonUnmarshalerType):
t, err := dec.peekRuneType()
if err != nil {
return err
}
var buf bytes.Buffer
if err := dec.scan(&buf); err != nil {
return err
}
obj := val.Addr().Interface().(jsonUnmarshaler)
if err := obj.UnmarshalJSON(buf.Bytes()); err != nil {
return dec.newTypeError(t.JSONType(), reflect.PointerTo(typ), err)
}
case val.CanAddr() && reflect.PointerTo(typ).Implements(textUnmarshalerType):
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
var buf bytes.Buffer
if err := dec.decodeString(reflect.PointerTo(typ), &buf); err != nil {
return err
}
obj := val.Addr().Interface().(encoding.TextUnmarshaler)
if err := obj.UnmarshalText(buf.Bytes()); err != nil {
return dec.newTypeError("string", reflect.PointerTo(typ), err)
}
default:
switch kind := typ.Kind(); kind {
case reflect.Bool:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
b, err := dec.decodeBool(typ)
if err != nil {
return err
}
val.SetBool(b)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
var buf strings.Builder
if err := dec.scanNumber(typ, &buf); err != nil {
return err
}
n, err := strconv.ParseInt(buf.String(), 10, kind2bits[kind])
if err != nil {
return dec.newTypeError("number "+buf.String(), typ, err)
}
val.SetInt(n)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
var buf strings.Builder
if err := dec.scanNumber(typ, &buf); err != nil {
return err
}
n, err := strconv.ParseUint(buf.String(), 10, kind2bits[kind])
if err != nil {
return dec.newTypeError("number "+buf.String(), typ, err)
}
val.SetUint(n)
case reflect.Float32, reflect.Float64:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
var buf strings.Builder
if err := dec.scanNumber(typ, &buf); err != nil {
return err
}
n, err := strconv.ParseFloat(buf.String(), kind2bits[kind])
if err != nil {
return dec.newTypeError("number "+buf.String(), typ, err)
}
val.SetFloat(n)
case reflect.String:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
var buf strings.Builder
if typ == numberType {
t, err := dec.peekRuneType()
if err != nil {
return err
}
if err := dec.scan(&buf); err != nil {
return err
}
if !t.IsNumber() {
return dec.newTypeError(t.JSONType(), typ,
fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number",
buf.String()))
}
val.SetString(buf.String())
} else {
if err := dec.decodeString(typ, &buf); err != nil {
return err
}
val.SetString(buf.String())
}
case reflect.Interface:
t, err := dec.peekRuneType()
if err != nil {
return err
}
if typ.NumMethod() > 0 {
return dec.newTypeError(t.JSONType(), typ, ErrDecodeNonEmptyInterface)
}
// If the interface stores a pointer, try to use the type information of the pointer.
if !val.IsNil() && val.Elem().Kind() == reflect.Pointer {
// Follow a chain of pointers until we find the first settable
// pointer (if any).
ptr := val.Elem()
for {
if ptr.CanSet() || ptr.IsNil() || ptr.Elem().Kind() != reflect.Pointer {
// We've reached the end of the line, good or bad.
break
}
ptr = ptr.Elem()
}
// ptr.Elem() != val
//
// Avoid the loop of an interface storing a pointer to its own
// address. We only need to worry about this at the leaf (and not
// in the loop) because the only way it's possible is if there's
// an interface in there, which'd break from the loop on its own.
//
// ptr.CanSet() || t != jsonparse.RuneTypeNullN
//
// We only need the pointer itself to be settable if we're
// decoding null.
if ptr.Elem() != val && (ptr.CanSet() || t != jsonparse.RuneTypeNullN) {
if err := dec.decode(ptr, false); err != nil {
return err
}
break
}
}
// Couldn't get type information from a pointer; fall back to untyped mode.
switch t {
case jsonparse.RuneTypeNullN:
if err := dec.decodeNull(); err != nil {
return err
}
val.Set(reflect.Zero(typ))
default:
v, err := dec.decodeAny()
if err != nil {
return err
}
val.Set(reflect.ValueOf(v))
}
case reflect.Struct:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
index := jsonstruct.IndexStruct(typ)
var nameBuf strings.Builder
return dec.decodeObject(typ, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
nameBuf.Reset()
return dec.decodeString(nil, &nameBuf)
}, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
name := nameBuf.String()
dec.structStackPush(typ, name)
defer dec.structStackPop()
idx, ok := index.ByName[name]
if !ok {
for oidx := range index.ByPos {
if strings.EqualFold(name, index.ByPos[oidx].Name) {
idx = oidx
ok = true
break
}
}
}
if !ok {
if dec.disallowUnknownFields {
return dec.newTypeError("", typ, fmt.Errorf("json: unknown field %q", name))
}
return dec.scan(fastio.Discard)
}
field := index.ByPos[idx]
fVal := val
for _, idx := range field.Path {
if fVal.Kind() == reflect.Pointer {
if fVal.IsNil() && !fVal.CanSet() { // https://golang.org/issue/21357
return dec.newTypeError("", fVal.Type().Elem(),
fmt.Errorf("json: cannot set embedded pointer to unexported struct: %v",
fVal.Type().Elem()))
}
t, err := dec.peekRuneType()
if err != nil {
return err
}
if t != jsonparse.RuneTypeNullN {
if fVal.IsNil() {
fVal.Set(reflect.New(fVal.Type().Elem()))
}
fVal = fVal.Elem()
}
}
fVal = fVal.Field(idx)
}
if field.Quote {
t, err := dec.peekRuneType()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeNullN:
if err := dec.decodeNull(); err != nil {
return err
}
switch fVal.Kind() {
// XXX: I can't justify this list, other than "it's what encoding/json
// does, but I don't understand their rationale".
case reflect.Interface, reflect.Pointer, reflect.Map, reflect.Slice:
fVal.Set(reflect.Zero(fVal.Type()))
default:
// do nothing???
}
case jsonparse.RuneTypeStringBeg:
// TODO: Figure out how to do this without buffering, have correct offsets.
var buf bytes.Buffer
if err := dec.decodeString(nil, &buf); err != nil {
return err
}
if err := NewDecoder(bytes.NewReader(buf.Bytes())).Decode(fVal.Addr().Interface()); err != nil {
if str := buf.String(); str != "null" {
return dec.newTypeError("", fVal.Type(),
fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v",
str, fVal.Type()))
}
}
default:
return dec.newTypeError(t.JSONType(), fVal.Type(),
fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v",
fVal.Type()))
}
return nil
} else {
return dec.decode(fVal, true)
}
})
case reflect.Map:
t, err := dec.peekRuneType()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeNullN:
if err := dec.decodeNull(); err != nil {
return err
}
val.Set(reflect.Zero(typ))
case jsonparse.RuneTypeObjectBeg:
if val.IsNil() {
val.Set(reflect.MakeMap(typ))
}
var nameBuf bytes.Buffer
var nameValPtr reflect.Value
return dec.decodeObject(typ, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
nameBuf.Reset()
if err := dec.decodeString(nil, &nameBuf); err != nil {
return err
}
nameValTyp := typ.Key()
nameValPtr = reflect.New(nameValTyp)
switch {
case reflect.PointerTo(nameValTyp).Implements(textUnmarshalerType):
obj := nameValPtr.Interface().(encoding.TextUnmarshaler)
if err := obj.UnmarshalText(nameBuf.Bytes()); err != nil {
return dec.newTypeError("string", reflect.PointerTo(nameValTyp), err)
}
default:
switch nameValTyp.Kind() {
case reflect.String:
nameValPtr.Elem().SetString(nameBuf.String())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
n, err := strconv.ParseInt(nameBuf.String(), 10, kind2bits[nameValTyp.Kind()])
if err != nil {
return dec.newTypeError("number "+nameBuf.String(), nameValTyp, err)
}
nameValPtr.Elem().SetInt(n)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
n, err := strconv.ParseUint(nameBuf.String(), 10, kind2bits[nameValTyp.Kind()])
if err != nil {
return dec.newTypeError("number "+nameBuf.String(), nameValTyp, err)
}
nameValPtr.Elem().SetUint(n)
default:
return dec.newTypeError("object", typ, &DecodeArgumentError{Type: nameValTyp})
}
}
return nil
}, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
dec.structStackPush(typ, nameValPtr.Elem())
defer dec.structStackPop()
fValPtr := reflect.New(typ.Elem())
if err := dec.decode(fValPtr.Elem(), false); err != nil {
return err
}
val.SetMapIndex(nameValPtr.Elem(), fValPtr.Elem())
return nil
})
default:
return dec.newTypeError(t.JSONType(), typ, nil)
}
case reflect.Slice:
t, err := dec.peekRuneType()
if err != nil {
return err
}
switch {
case typ.Elem().Kind() == reflect.Uint8 && !(t == jsonparse.RuneTypeArrayBeg && (false ||
reflect.PointerTo(typ.Elem()).Implements(decodableType) ||
reflect.PointerTo(typ.Elem()).Implements(jsonUnmarshalerType) ||
reflect.PointerTo(typ.Elem()).Implements(textUnmarshalerType))):
switch t {
case jsonparse.RuneTypeNullN:
if err := dec.decodeNull(); err != nil {
return err
}
val.Set(reflect.Zero(typ))
case jsonparse.RuneTypeStringBeg:
if typ.Elem() == byteType {
var buf bytes.Buffer
if err := dec.decodeString(typ, base64dec.NewBase64Decoder(&buf)); err != nil {
return err
}
val.Set(reflect.ValueOf(buf.Bytes()))
} else {
// TODO: Surely there's a better way. At the very least, we should
// avoid buffering.
var buf bytes.Buffer
if err := dec.decodeString(typ, base64dec.NewBase64Decoder(&buf)); err != nil {
return err
}
bs := buf.Bytes()
val.Set(reflect.MakeSlice(typ, len(bs), len(bs)))
for i := 0; i < len(bs); i++ {
val.Index(i).Set(reflect.ValueOf(bs[i]).Convert(typ.Elem()))
}
}
default:
return dec.newTypeError(t.JSONType(), typ, nil)
}
default:
switch t {
case jsonparse.RuneTypeNullN:
if err := dec.decodeNull(); err != nil {
return err
}
val.Set(reflect.Zero(typ))
case jsonparse.RuneTypeArrayBeg:
if val.IsNil() {
val.Set(reflect.MakeSlice(typ, 0, 0))
}
if val.Len() > 0 {
val.Set(val.Slice(0, 0))
}
i := 0
return dec.decodeArray(typ, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
dec.structStackPush(typ, i)
defer dec.structStackPop()
mValPtr := reflect.New(typ.Elem())
if err := dec.decode(mValPtr.Elem(), false); err != nil {
return err
}
val.Set(reflect.Append(val, mValPtr.Elem()))
i++
return nil
})
default:
return dec.newTypeError(t.JSONType(), typ, nil)
}
}
case reflect.Array:
if ok, err := dec.maybeDecodeNull(nullOK); ok {
return err
}
i := 0
n := val.Len()
if err := dec.decodeArray(typ, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
dec.structStackPush(typ, i)
defer dec.structStackPop()
if i < n {
mValPtr := reflect.New(typ.Elem())
if err := dec.decode(mValPtr.Elem(), false); err != nil {
return err
}
val.Index(i).Set(mValPtr.Elem())
} else {
if err := dec.scan(fastio.Discard); err != nil {
return err
}
}
i++
return nil
}); err != nil {
return err
}
for ; i < n; i++ {
val.Index(i).Set(reflect.Zero(typ.Elem()))
}
case reflect.Pointer:
t, err := dec.peekRuneType()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeNullN:
if err := dec.decodeNull(); err != nil {
return err
}
val.Set(reflect.Zero(typ))
default:
if val.IsNil() {
val.Set(reflect.New(typ.Elem()))
}
return dec.decode(val.Elem(), false)
}
default:
return dec.newTypeError("", typ, fmt.Errorf("unsupported type (kind=%v)", typ.Kind()))
}
}
return nil
}
func (dec *Decoder) scan(out fastio.RuneWriter) *DecodeError {
dec.io.PushReadBarrier()
for {
c, t, err := dec.readRune()
if err != nil {
return err
}
if t == jsonparse.RuneTypeEOF {
break
}
_, _ = out.WriteRune(c)
}
dec.io.PopReadBarrier()
return nil
}
func (dec *Decoder) scanNumber(gTyp reflect.Type, out fastio.RuneWriter) *DecodeError {
t, err := dec.peekRuneType()
if err != nil {
return err
}
if !t.IsNumber() {
return dec.newTypeError(t.JSONType(), gTyp, nil)
}
return dec.scan(out)
}
func (dec *Decoder) decodeAny() (any, *DecodeError) {
t, err := dec.peekRuneType()
if err != nil {
return nil, err
}
switch t {
case jsonparse.RuneTypeObjectBeg:
ret := make(map[string]any)
typ := reflect.TypeOf(ret)
var nameBuf strings.Builder
if err := dec.decodeObject(typ, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
nameBuf.Reset()
return dec.decodeString(nil, &nameBuf)
}, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
name := nameBuf.String()
dec.structStackPush(typ, name)
defer dec.structStackPop()
v, err := dec.decodeAny()
if err != nil {
return err
}
ret[name] = v
return nil
}); err != nil {
return nil, err
}
return ret, nil
case jsonparse.RuneTypeArrayBeg:
ret := []any{}
typ := reflect.TypeOf(ret)
if err := dec.decodeArray(typ, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
dec.structStackPush(typ, len(ret))
defer dec.structStackPop()
v, err := dec.decodeAny()
if err != nil {
return err
}
ret = append(ret, v)
return nil
}); err != nil {
return nil, err
}
return ret, nil
case jsonparse.RuneTypeStringBeg:
var buf strings.Builder
if err := dec.decodeString(nil, &buf); err != nil {
return nil, err
}
return buf.String(), nil
case jsonparse.RuneTypeNumberIntNeg, jsonparse.RuneTypeNumberIntZero, jsonparse.RuneTypeNumberIntDig:
var buf strings.Builder
if err := dec.scan(&buf); err != nil {
return nil, err
}
num := Number(buf.String())
if dec.useNumber {
return num, nil
}
f64, err := num.Float64()
if err != nil {
return nil, dec.newTypeError("number "+buf.String(), float64Type, err)
}
return f64, nil
case jsonparse.RuneTypeTrueT, jsonparse.RuneTypeFalseF:
return dec.decodeBool(nil)
case jsonparse.RuneTypeNullN:
return nil, dec.decodeNull()
default:
panic(fmt.Errorf("should not happen: unexpected runeType at beginning of value: %v", t))
}
}
// DecodeObject is a helper function to ease implementing the
// Decodable interface; allowing the lowmemjson package to handle
// decoding the object syntax, while the Decodable only needs to
// handle decoding the keys and values within the object.
//
// Outside of implementing Decodable.DecodeJSON methods, callers
// should instead simply use NewDecoder(r).Decode(&val) rather than
// attempting to call DecodeObject directly.
func DecodeObject(r io.RuneScanner, decodeKey, decodeVal func(io.RuneScanner) error) error {
var dec *Decoder
if dr, ok := r.(*decRuneScanner); ok {
dec = dr.dec
} else {
dec = NewDecoder(r)
}
dec.posStackPush()
defer dec.posStackPop()
if err := dec.decodeObject(nil,
func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
return dec.withLimitingScanner(func(l io.RuneScanner) *DecodeError {
if err := decodeKey(l); err != nil {
// TODO: Find a better Go type to use than `nil`.
return dec.newTypeError("string", nil, err)
}
if _, _, err := l.ReadRune(); err != io.EOF {
// TODO: Find a better Go type to use than `nil`.
return dec.newTypeError("string", nil, fmt.Errorf("did not consume entire string"))
}
return nil
})
},
func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
t, err := dec.peekRuneType()
if err != nil {
return err
}
return dec.withLimitingScanner(func(l io.RuneScanner) *DecodeError {
if err := decodeVal(l); err != nil {
// TODO: Find a better Go type to use than `nil`.
return dec.newTypeError(t.JSONType(), nil, err)
}
if _, _, err := l.ReadRune(); err != io.EOF {
// TODO: Find a better Go type to use than `nil`.
return dec.newTypeError(t.JSONType(), nil, fmt.Errorf("did not consume entire %s", t.JSONType()))
}
return nil
})
}); err != nil {
return err
}
return nil
}
func (dec *Decoder) decodeObject(gTyp reflect.Type, decodeKey, decodeVal func() *DecodeError) *DecodeError {
if err := dec.expectRuneType('{', jsonparse.RuneTypeObjectBeg, gTyp); err != nil {
return err
}
_, t, err := dec.readRune()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeObjectEnd:
return nil
case jsonparse.RuneTypeStringBeg:
decodeMember:
dec.unreadRune()
if err := decodeKey(); err != nil {
return err
}
if err := dec.expectRune(':', jsonparse.RuneTypeObjectColon); err != nil {
return err
}
if err := decodeVal(); err != nil {
return err
}
_, t, err := dec.readRune()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeObjectComma:
if err := dec.expectRune('"', jsonparse.RuneTypeStringBeg); err != nil {
return err
}
goto decodeMember
case jsonparse.RuneTypeObjectEnd:
return nil
default:
panic(fmt.Errorf("should not happen: unexpected runeType after k/v pair in object: %v", t))
}
default:
panic(fmt.Errorf("should not happen: unexpected runeType after opening '{' of object: %v", t))
}
}
// DecodeArray is a helper function to ease implementing the Decoder
// interface; allowing the lowmemjson package to handle decoding the
// array syntax, while the Decodable only needs to handle decoding
// members within the array.
//
// Outside of implementing Decodable.DecodeJSON methods, callers
// should instead simply use NewDecoder(r).Decode(&val) rather than
// attempting to call DecodeArray directly.
func DecodeArray(r io.RuneScanner, decodeMember func(r io.RuneScanner) error) error {
var dec *Decoder
if dr, ok := r.(*decRuneScanner); ok {
dec = dr.dec
} else {
dec = NewDecoder(r)
}
dec.posStackPush()
defer dec.posStackPop()
if err := dec.decodeArray(nil, func() *DecodeError {
dec.posStackPush()
defer dec.posStackPop()
t, err := dec.peekRuneType()
if err != nil {
return err
}
return dec.withLimitingScanner(func(l io.RuneScanner) *DecodeError {
if err := decodeMember(l); err != nil {
// TODO: Find a better Go type to use than `nil`.
return dec.newTypeError(t.JSONType(), nil, err)
}
if _, _, err := l.ReadRune(); err != io.EOF {
// TODO: Find a better Go type to use than `nil`.
return dec.newTypeError(t.JSONType(), nil, fmt.Errorf("did not consume entire %s", t.JSONType()))
}
return nil
})
}); err != nil {
return err
}
return nil
}
func (dec *Decoder) decodeArray(gTyp reflect.Type, decodeMember func() *DecodeError) *DecodeError {
if err := dec.expectRuneType('[', jsonparse.RuneTypeArrayBeg, gTyp); err != nil {
return err
}
_, t, err := dec.readRune()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeArrayEnd:
return nil
default:
dec.unreadRune()
decodeNextMember:
if err := decodeMember(); err != nil {
return err
}
_, t, err := dec.readRune()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeArrayComma:
goto decodeNextMember
case jsonparse.RuneTypeArrayEnd:
return nil
default:
panic(fmt.Errorf("should not happen: unexpected runeType after array item: %v", t))
}
}
}
func (dec *Decoder) decodeString(gTyp reflect.Type, out fastio.RuneWriter) *DecodeError {
if err := dec.expectRuneType('"', jsonparse.RuneTypeStringBeg, gTyp); err != nil {
return err
}
var uhex [4]byte
for {
c, t, err := dec.readRune()
if err != nil {
return err
}
switch t {
case jsonparse.RuneTypeStringChar:
_, _ = out.WriteRune(c)
case jsonparse.RuneTypeStringEsc, jsonparse.RuneTypeStringEscU:
// do nothing
case jsonparse.RuneTypeStringEsc1:
switch c {
case '"':
_, _ = out.WriteRune('"')
case '\\':
_, _ = out.WriteRune('\\')
case '/':
_, _ = out.WriteRune('/')
case 'b':
_, _ = out.WriteRune('\b')
case 'f':
_, _ = out.WriteRune('\f')
case 'n':
_, _ = out.WriteRune('\n')
case 'r':
_, _ = out.WriteRune('\r')
case 't':
_, _ = out.WriteRune('\t')
default:
panic(fmt.Errorf("should not happen: unexpected rune after backslash: %q", c))
}
case jsonparse.RuneTypeStringEscUA:
uhex[0], _ = jsonparse.HexToInt(c)
case jsonparse.RuneTypeStringEscUB:
uhex[1], _ = jsonparse.HexToInt(c)
case jsonparse.RuneTypeStringEscUC:
uhex[2], _ = jsonparse.HexToInt(c)
case jsonparse.RuneTypeStringEscUD:
uhex[3], _ = jsonparse.HexToInt(c)
c = 0 |
rune(uhex[0])<<12 |
rune(uhex[1])<<8 |
rune(uhex[2])<<4 |
rune(uhex[3])<<0
handleUnicode:
if utf16.IsSurrogate(c) {
t, err := dec.peekRuneType()
if err != nil {
return err
}
if t != jsonparse.RuneTypeStringEsc {
_, _ = out.WriteRune(utf8.RuneError)
break
}
if err := dec.expectRune('\\', jsonparse.RuneTypeStringEsc); err != nil {
return err
}
t, err = dec.peekRuneType()
if err != nil {
return err
}
if t != jsonparse.RuneTypeStringEscU {
_, _ = out.WriteRune(utf8.RuneError)
break
}
if err := dec.expectRune('u', jsonparse.RuneTypeStringEscU); err != nil {
return err
}
b, _, err := dec.readRune()
if err != nil {
return err
}
uhex[0], _ = jsonparse.HexToInt(b)
b, _, err = dec.readRune()
if err != nil {
return err
}
uhex[1], _ = jsonparse.HexToInt(b)
b, _, err = dec.readRune()
if err != nil {
return err
}
uhex[2], _ = jsonparse.HexToInt(b)
b, _, err = dec.readRune()
if err != nil {
return err
}
uhex[3], _ = jsonparse.HexToInt(b)
c2 := 0 |
rune(uhex[0])<<12 |
rune(uhex[1])<<8 |
rune(uhex[2])<<4 |
rune(uhex[3])<<0
d := utf16.DecodeRune(c, c2)
if d == utf8.RuneError {
_, _ = out.WriteRune(utf8.RuneError)
c = c2
goto handleUnicode
}
_, _ = out.WriteRune(d)
} else {
_, _ = out.WriteRune(c)
}
case jsonparse.RuneTypeStringEnd:
return nil
default:
panic(fmt.Errorf("should not happen: unexpected runeType in string: %v", t))
}
}
}
func (dec *Decoder) decodeBool(gTyp reflect.Type) (bool, *DecodeError) {
c, t, err := dec.readRune()
if err != nil {
return false, err
}
switch c {
case 't':
if err := dec.expectRune('r', jsonparse.RuneTypeTrueR); err != nil {
return false, err
}
if err := dec.expectRune('u', jsonparse.RuneTypeTrueU); err != nil {
return false, err
}
if err := dec.expectRune('e', jsonparse.RuneTypeTrueE); err != nil {
return false, err
}
return true, nil
case 'f':
if err := dec.expectRune('a', jsonparse.RuneTypeFalseA); err != nil {
return false, err
}
if err := dec.expectRune('l', jsonparse.RuneTypeFalseL); err != nil {
return false, err
}
if err := dec.expectRune('s', jsonparse.RuneTypeFalseS); err != nil {
return false, err
}
if err := dec.expectRune('e', jsonparse.RuneTypeFalseE); err != nil {
return false, err
}
return false, nil
default:
return false, dec.newTypeError(t.JSONType(), gTyp, nil)
}
}
func (dec *Decoder) decodeNull() *DecodeError {
if err := dec.expectRune('n', jsonparse.RuneTypeNullN); err != nil {
return err
}
if err := dec.expectRune('u', jsonparse.RuneTypeNullU); err != nil {
return err
}
if err := dec.expectRune('l', jsonparse.RuneTypeNullL1); err != nil {
return err
}
if err := dec.expectRune('l', jsonparse.RuneTypeNullL2); err != nil {
return err
}
return nil
}
func (dec *Decoder) maybeDecodeNull(nullOK bool) (ok bool, err *DecodeError) {
if nullOK {
t, err := dec.peekRuneType()
if err != nil {
return true, err
}
if t == jsonparse.RuneTypeNullN {
return true, dec.decodeNull()
}
}
return false, nil
}