From 921eeab75a87d07eaf9cec57dcdc8a3c276f291a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Feb 2023 11:01:17 -0700 Subject: compat/json: Use fastio.Discard instead of io.Discard --- compat/json/compat.go | 3 ++- compat/json/testcompat_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/compat/json/compat.go b/compat/json/compat.go index c96470d..0b86732 100644 --- a/compat/json/compat.go +++ b/compat/json/compat.go @@ -15,6 +15,7 @@ import ( "strconv" "git.lukeshu.com/go/lowmemjson" + "git.lukeshu.com/go/lowmemjson/internal/fastio" ) //nolint:stylecheck // ST1021 False positive; these aren't comments on individual types. @@ -172,7 +173,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { } func Valid(data []byte) bool { - formatter := lowmemjson.NewReEncoder(io.Discard, lowmemjson.ReEncoderConfig{ + formatter := lowmemjson.NewReEncoder(fastio.Discard, lowmemjson.ReEncoderConfig{ Compact: true, }) _, err := formatter.Write(data) diff --git a/compat/json/testcompat_test.go b/compat/json/testcompat_test.go index 42cbf5c..c186678 100644 --- a/compat/json/testcompat_test.go +++ b/compat/json/testcompat_test.go @@ -7,10 +7,10 @@ package json import ( "bytes" "encoding/json" - "io" _ "unsafe" "git.lukeshu.com/go/lowmemjson" + "git.lukeshu.com/go/lowmemjson/internal/fastio" "git.lukeshu.com/go/lowmemjson/internal/jsonparse" "git.lukeshu.com/go/lowmemjson/internal/jsonstring" "git.lukeshu.com/go/lowmemjson/internal/jsonstruct" @@ -25,7 +25,7 @@ var ( type scanner = lowmemjson.ReEncoderConfig func checkValid(in []byte, scan *lowmemjson.ReEncoderConfig) error { - return reencode(io.Discard, in, *scan) + return reencode(fastio.Discard, in, *scan) } func isValidNumber(s string) bool { -- cgit v1.2.3-2-g168b From a0e44530509d3b342b8011ac4467d957350f5ffa Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 15 Feb 2023 15:05:23 -0700 Subject: Avoid io.Writer causing buffers to escape to the heap --- borrowed_misc.go | 4 +++- internal/base64dec/base64.go | 3 ++- internal/fastio/allwriter.go | 8 +++++--- internal/fastio/noescape/noescape.go | 30 ++++++++++++++++++++++++++++++ internal/jsonstring/encode_string.go | 5 +++-- 5 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 internal/fastio/noescape/noescape.go diff --git a/borrowed_misc.go b/borrowed_misc.go index 52f4a12..e4cda6d 100644 --- a/borrowed_misc.go +++ b/borrowed_misc.go @@ -11,6 +11,8 @@ import ( "math" "reflect" "strconv" + + "git.lukeshu.com/go/lowmemjson/internal/fastio/noescape" ) // isEmptyValue is borrowed from encode.go. @@ -66,7 +68,7 @@ func encodeFloat(w io.Writer, bits int, v reflect.Value) error { } } - if _, err := w.Write(b); err != nil { + if _, err := noescape.Write(w, b); err != nil { return err } return nil diff --git a/internal/base64dec/base64.go b/internal/base64dec/base64.go index dcb4b1c..0d278ad 100644 --- a/internal/base64dec/base64.go +++ b/internal/base64dec/base64.go @@ -10,6 +10,7 @@ import ( "strings" "git.lukeshu.com/go/lowmemjson/internal/fastio" + "git.lukeshu.com/go/lowmemjson/internal/fastio/noescape" ) type base64Decoder struct { @@ -84,7 +85,7 @@ func (dec *base64Decoder) decodeTuple(a, b, c, d byte) error { decoded[1] = byte(val >> 8) decoded[2] = byte(val >> 0) - _, err := dec.dst.Write(decoded[:decodedLen]) + _, err := noescape.Write(dec.dst, decoded[:decodedLen]) return err } diff --git a/internal/fastio/allwriter.go b/internal/fastio/allwriter.go index 9de8fdc..c587531 100644 --- a/internal/fastio/allwriter.go +++ b/internal/fastio/allwriter.go @@ -7,6 +7,8 @@ package fastio import ( "io" "unicode/utf8" + + "git.lukeshu.com/go/lowmemjson/internal/fastio/noescape" ) // interfaces ///////////////////////////////////////////////////////////////// @@ -28,18 +30,18 @@ type AllWriter interface { func WriteByte(w io.Writer, b byte) error { var buf [1]byte buf[0] = b - _, err := w.Write(buf[:]) + _, err := noescape.Write(w, buf[:]) return err } func WriteRune(w io.Writer, r rune) (int, error) { var buf [utf8.UTFMax]byte n := utf8.EncodeRune(buf[:], r) - return w.Write(buf[:n]) + return noescape.Write(w, buf[:n]) } func WriteString(w io.Writer, s string) (int, error) { - return w.Write([]byte(s)) + return noescape.Write(w, []byte(s)) } // wrappers /////////////////////////////////////////////////////////////////// diff --git a/internal/fastio/noescape/noescape.go b/internal/fastio/noescape/noescape.go new file mode 100644 index 0000000..02d25b5 --- /dev/null +++ b/internal/fastio/noescape/noescape.go @@ -0,0 +1,30 @@ +// Copyright (C) 2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package noescape + +import ( + "io" + _ "unsafe" +) + +//go:noescape +//go:linkname Write io.Writer.Write +func Write(w io.Writer, dat []byte) (int, error) + +//go:noescape +//go:linkname WriteString io.StringWriter.WriteString +func WriteString(w io.Writer, dat string) (int, error) + +//go:noescape +//go:linkname WriteAt io.WriterAt.WriteAt +func WriteAt(w io.WriterAt, dat []byte, off int64) (int, error) + +//go:noescape +//go:linkname Read io.Reader.Read +func Read(w io.Reader, dat []byte) (int, error) + +//go:noescape +//go:linkname ReadAt io.ReaderAt.ReadAt +func ReadAt(w io.WriterAt, dat []byte, off int64) (int, error) diff --git a/internal/jsonstring/encode_string.go b/internal/jsonstring/encode_string.go index 1b0c68a..fec2cc0 100644 --- a/internal/jsonstring/encode_string.go +++ b/internal/jsonstring/encode_string.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" "git.lukeshu.com/go/lowmemjson/internal/fastio" + "git.lukeshu.com/go/lowmemjson/internal/fastio/noescape" ) // BackslashEscapeMode is describe in the main lowmemjson package @@ -35,7 +36,7 @@ func writeStringUnicodeEscape(w io.Writer, c rune) error { alphabet[(c>>4)&0xf], alphabet[(c>>0)&0xf], } - _, err := w.Write(buf[:]) + _, err := noescape.Write(w, buf[:]) return err } @@ -58,7 +59,7 @@ func writeStringShortEscape(w io.Writer, c rune) error { panic(fmt.Errorf("should not happen: writeStringShortEscape called with invalid rune: %q", c)) } buf := [2]byte{'\\', b} - _, err := w.Write(buf[:]) + _, err := noescape.Write(w, buf[:]) return err } -- cgit v1.2.3-2-g168b From 30e0ee4ced0be18b4df91674fd6f073793b61fe9 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 15 Feb 2023 18:58:51 -0700 Subject: encode: Avoid allocations when formatting integers --- encode.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/encode.go b/encode.go index 2e10134..38a2e93 100644 --- a/encode.go +++ b/encode.go @@ -228,7 +228,12 @@ func encode(w *ReEncoder, val reflect.Value, escaper BackslashEscaper, quote boo return err } } - if _, err := w.WriteString(strconv.FormatInt(val.Int(), 10)); err != nil { + // MaxInt64 = 9223372036854775807 + // MinInt64 = -9223372036854775808 + // 0 1 2 + // 12345678901234567890 + var buf [20]byte + if _, err := w.Write(strconv.AppendInt(buf[:0], val.Int(), 10)); err != nil { return err } if quote { @@ -242,7 +247,11 @@ func encode(w *ReEncoder, val reflect.Value, escaper BackslashEscaper, quote boo return err } } - if _, err := w.WriteString(strconv.FormatUint(val.Uint(), 10)); err != nil { + // MaxUint64 = 18446744073709551615 + // 0 1 2 + // 12345678901234567890 + var buf [20]byte + if _, err := w.Write(strconv.AppendUint(buf[:0], val.Uint(), 10)); err != nil { return err } if quote { -- cgit v1.2.3-2-g168b From 0d23080e1f2af81d8d4656c3b72791b26f52f361 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 15 Feb 2023 23:45:07 -0700 Subject: reencode: utf8.RuneLen is showing up in the profile --- reencode.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/reencode.go b/reencode.go index f18888c..d8cdb71 100644 --- a/reencode.go +++ b/reencode.go @@ -165,14 +165,14 @@ func (enc *ReEncoder) Write(p []byte) (int, error) { c, size := utf8.DecodeRune(enc.buf[:]) n += size - enc.bufLen enc.bufLen = 0 - enc.handleRune(c) + enc.handleRune(c, size) if enc.err != nil { return 0, enc.err } } for utf8.FullRune(p[n:]) { c, size := utf8.DecodeRune(p[n:]) - enc.handleRune(c) + enc.handleRune(c, size) if enc.err != nil { return n, enc.err } @@ -194,14 +194,14 @@ func (enc *ReEncoder) WriteString(p string) (int, error) { c, size := utf8.DecodeRune(enc.buf[:]) n += size - enc.bufLen enc.bufLen = 0 - enc.handleRune(c) + enc.handleRune(c, size) if enc.err != nil { return 0, enc.err } } for utf8.FullRuneInString(p[n:]) { c, size := utf8.DecodeRuneInString(p[n:]) - enc.handleRune(c) + enc.handleRune(c, size) if enc.err != nil { return n, enc.err } @@ -253,7 +253,7 @@ func (enc *ReEncoder) Close() error { return nil } -func (enc *ReEncoder) handleRune(c rune) { +func (enc *ReEncoder) handleRune(c rune, size int) { rehandle: t, err := enc.par.HandleRune(c) if err != nil { @@ -283,7 +283,7 @@ rehandle: } } - enc.inputPos += int64(utf8.RuneLen(c)) + enc.inputPos += int64(size) } // semi-public API ///////////////////////////////////////////////////////////// -- cgit v1.2.3-2-g168b