Repository navigation
Render nested scalars without encoding/json #527
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
c86fe56
Render nested scalars without encoding/json
vuanhphung 0d490a0
Speed up jsonfmt string scan and fallback
vuanhphung cd7d3e1
Fast-path UTF-8 strings and reuse scratch buffers
vuanhphung 80e28d9
Pin the jsonfmt fast path and drop Marshal
vuanhphung File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| // Package jsonfmt renders scalar values as JSON, byte-identical to | ||
| // encoding/json.Marshal, for the nested-type renderers shared by the Thrift | ||
| // (arrowbased) and kernel (arrowscan) result paths. | ||
| // | ||
| // Those renderers marshal one scalar per nested element, so per-call overhead | ||
| // dominates. Go 1.27 routes encoding/json through the json/v2 engine by default, | ||
| // which made that per-call cost markedly higher; formatting the common scalar | ||
| // types directly avoids it. Anything not handled here falls back to json.Marshal, | ||
| // so output never diverges from it. | ||
| package jsonfmt | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "math" | ||
| "strconv" | ||
| "unicode/utf8" | ||
| ) | ||
|
|
||
| // Append appends the JSON encoding of v to dst, exactly as json.Marshal(v) | ||
| // would render it, and returns the extended slice. | ||
| func Append(dst []byte, v any) ([]byte, error) { | ||
| switch t := v.(type) { | ||
| case nil: | ||
| return append(dst, "null"...), nil | ||
| case bool: | ||
| return strconv.AppendBool(dst, t), nil | ||
| case int: | ||
| return strconv.AppendInt(dst, int64(t), 10), nil | ||
| case int8: | ||
| return strconv.AppendInt(dst, int64(t), 10), nil | ||
| case int16: | ||
| return strconv.AppendInt(dst, int64(t), 10), nil | ||
| case int32: | ||
| return strconv.AppendInt(dst, int64(t), 10), nil | ||
| case int64: | ||
| return strconv.AppendInt(dst, t, 10), nil | ||
| case uint8: | ||
| return strconv.AppendUint(dst, uint64(t), 10), nil | ||
| case uint16: | ||
| return strconv.AppendUint(dst, uint64(t), 10), nil | ||
| case uint32: | ||
| return strconv.AppendUint(dst, uint64(t), 10), nil | ||
| case uint64: | ||
| return strconv.AppendUint(dst, t, 10), nil | ||
| case float32: | ||
| if !isFinite(float64(t)) { | ||
| break | ||
| } | ||
| return appendFloat(dst, float64(t), 32), nil | ||
| case float64: | ||
| if !isFinite(t) { | ||
| break | ||
| } | ||
| return appendFloat(dst, t, 64), nil | ||
| case string: | ||
| if needsEscape(t) { | ||
| break | ||
| } | ||
| dst = append(dst, '"') | ||
| dst = append(dst, t...) | ||
| return append(dst, '"'), nil | ||
| } | ||
| // NaN/Inf (json.Marshal's error), strings needing escapes, []byte (base64), | ||
| // and any other type. | ||
| b, err := json.Marshal(v) | ||
| if err != nil { | ||
| return dst, err | ||
| } | ||
| if len(dst) == 0 { | ||
| return b, nil // skip copying json.Marshal's fresh slice | ||
| } | ||
| return append(dst, b...), nil | ||
| } | ||
|
|
||
| func isFinite(f float64) bool { | ||
| return !math.IsInf(f, 0) && !math.IsNaN(f) | ||
| } | ||
|
|
||
| // appendFloat mirrors encoding/json's ES6-style float formatting: plain decimal | ||
| // unless the magnitude is below 1e-6 or at least 1e21, with exponents unpadded. | ||
| func appendFloat(dst []byte, f float64, bits int) []byte { | ||
| abs := math.Abs(f) | ||
| format := byte('f') | ||
| // Compare at the value's own width so float32 cutoffs match exactly. | ||
| if abs != 0 { | ||
| if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) { | ||
| format = 'e' | ||
| } | ||
| } | ||
| dst = strconv.AppendFloat(dst, f, format, -1, bits) | ||
| if format == 'e' { | ||
| // e-09 -> e-9 | ||
| n := len(dst) | ||
| if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' { | ||
| dst[n-2] = dst[n-1] | ||
| dst = dst[:n-1] | ||
| } | ||
| } | ||
| return dst | ||
| } | ||
|
|
||
| // needsEscape reports whether json.Marshal would escape any part of s. It copies | ||
| // ASCII verbatim except control characters, the quote/backslash and the | ||
| // HTML-escaped <, >, &; and valid multi-byte UTF-8 verbatim except U+2028/U+2029. | ||
| // Everything else (including invalid UTF-8) takes the fallback. | ||
| func needsEscape(s string) bool { | ||
| for i := 0; i < len(s); i++ { | ||
| if !plainASCII[s[i]] { | ||
| if s[i] < utf8.RuneSelf { | ||
| return true | ||
| } | ||
| return needsEscapeUTF8(s[i:]) | ||
| } | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| // needsEscapeUTF8 is needsEscape for the rest of s from its first non-ASCII byte. | ||
| func needsEscapeUTF8(s string) bool { | ||
| for i := 0; i < len(s); { | ||
| if c := s[i]; c < utf8.RuneSelf { | ||
| if !plainASCII[c] { | ||
| return true | ||
| } | ||
| i++ | ||
| continue | ||
| } | ||
| r, size := utf8.DecodeRuneInString(s[i:]) | ||
| if r == utf8.RuneError && size == 1 || r == 0x2028 || r == 0x2029 { | ||
| return true | ||
| } | ||
| i += size | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| // plainASCII marks the ASCII bytes json.Marshal copies verbatim. A table lookup | ||
| // keeps the scan faster than json.Marshal's own on long strings. | ||
| var plainASCII = func() (t [256]bool) { | ||
| for c := 0x20; c <= 0x7f; c++ { | ||
| t[c] = true | ||
| } | ||
| for _, c := range `"\<>&` { | ||
| t[c] = false | ||
| } | ||
| return t | ||
| }() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| package jsonfmt | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "math" | ||
| "math/rand" | ||
| "strings" | ||
| "testing" | ||
| "unicode/utf8" | ||
| ) | ||
|
|
||
| // assertParity fails unless Append(nil, v) matches json.Marshal(v) | ||
| // byte-for-byte, including any error. | ||
| func assertParity(t *testing.T, v any) { | ||
| t.Helper() | ||
| want, wantErr := json.Marshal(v) | ||
| got, gotErr := Append(nil, v) | ||
| if (wantErr != nil) != (gotErr != nil) || wantErr != nil && gotErr.Error() != wantErr.Error() { | ||
| t.Fatalf("%#v: error mismatch: got %v, want %v", v, gotErr, wantErr) | ||
| } | ||
| if wantErr == nil && string(got) != string(want) { | ||
| t.Fatalf("%#v: got %s, want %s", v, got, want) | ||
| } | ||
| } | ||
|
|
||
| func TestMarshalMatchesEncodingJSON(t *testing.T) { | ||
| values := []any{ | ||
| nil, true, false, | ||
| 0, -1, math.MaxInt, math.MinInt, | ||
| int8(math.MinInt8), int8(math.MaxInt8), int16(math.MinInt16), int32(math.MinInt32), int64(math.MinInt64), int64(math.MaxInt64), | ||
| uint8(255), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), | ||
| 0.0, math.Copysign(0, -1), 1.0, -1.5, 3.14, 0.1, 1e20, 1e21, 1e-6, 1e-7, 9.999999e-7, 123456789.123456789, | ||
| math.MaxFloat64, math.SmallestNonzeroFloat64, -math.MaxFloat64, | ||
| float32(0), float32(math.Copysign(0, -1)), float32(3.14), float32(0.1), float32(1e21), float32(1e-6), float32(1e-7), | ||
| float32(math.MaxFloat32), float32(math.SmallestNonzeroFloat32), | ||
| math.NaN(), math.Inf(1), math.Inf(-1), float32(math.NaN()), float32(math.Inf(-1)), | ||
| "", "alpha", "with space", "a\"b", `back\slash`, "<tag>", "a&b", "tab\there", "nl\n", "\x00", "\x7f", | ||
| "héllo", "日本語", " ", "\xff\xfe", "emoji 😀", | ||
| []byte("abc"), []byte(nil), | ||
| } | ||
| for _, v := range values { | ||
| assertParity(t, v) | ||
| } | ||
| } | ||
|
|
||
| // Every single-byte string, so the escape set can't drift from json.Marshal's. | ||
| func TestMarshalStringBytes(t *testing.T) { | ||
| for c := 0; c < 256; c++ { | ||
| assertParity(t, string([]byte{byte(c)})) | ||
| assertParity(t, "x"+string([]byte{byte(c)})+"y") | ||
| } | ||
| } | ||
|
|
||
| // Every rune (surrogates encode as U+FFFD), so the UTF-8 fast path can't drift | ||
| // from json.Marshal. | ||
| func TestMarshalEveryRune(t *testing.T) { | ||
| for r := rune(0); r <= utf8.MaxRune; r++ { | ||
| assertParity(t, "x"+string(r)) | ||
| } | ||
| } | ||
|
|
||
| // Random mixes of ASCII, multi-byte runes, U+2028/U+2029 and raw invalid bytes. | ||
| func TestMarshalRandomUnicodeStrings(t *testing.T) { | ||
| r := rand.New(rand.NewSource(3)) //nolint:gosec // G404: deterministic test inputs, not security | ||
| pieces := []string{"a", "Z", " ", "<", "&", "\"", "\\", "\n", "\x7f"} | ||
| for _, cp := range []rune{0xe9, 0xdf, 0x65e5, 0x1f600, 0x2028, 0x2029, 0xa0, 0xfeff, 0xffff} { | ||
| pieces = append(pieces, string(cp)) | ||
| } | ||
| for i := 0; i < 100000; i++ { | ||
| var sb strings.Builder | ||
| for j, n := 0, r.Intn(12); j < n; j++ { | ||
| switch r.Intn(4) { | ||
| case 0: | ||
| var raw [1]byte // possibly invalid UTF-8 | ||
| _, _ = r.Read(raw[:]) | ||
| sb.Write(raw[:]) | ||
| case 1: | ||
| sb.WriteRune(r.Int31n(utf8.MaxRune + 1)) | ||
| default: | ||
| sb.WriteString(pieces[r.Intn(len(pieces))]) | ||
| } | ||
| } | ||
| assertParity(t, sb.String()) | ||
| } | ||
| } | ||
|
|
||
| // Random bit patterns reach exponent cutoffs, subnormals, and rounding cases a | ||
| // hand-picked list misses. | ||
| func TestMarshalRandomFloats(t *testing.T) { | ||
| r := rand.New(rand.NewSource(1)) //nolint:gosec // G404: deterministic test inputs, not security | ||
| for i := 0; i < 200000; i++ { | ||
| assertParity(t, math.Float64frombits(r.Uint64())) | ||
| assertParity(t, math.Float32frombits(r.Uint32())) | ||
| assertParity(t, r.NormFloat64()*math.Pow(10, float64(r.Intn(50)-25))) | ||
| assertParity(t, float32(r.NormFloat64()*math.Pow(10, float64(r.Intn(30)-15)))) | ||
| } | ||
| } | ||
|
|
||
| func TestAppendKeepsPrefix(t *testing.T) { | ||
| got, err := Append([]byte("["), int32(7)) | ||
| if err != nil || string(got) != "[7" { | ||
| t.Fatalf("got %q, %v", got, err) | ||
| } | ||
| got, err = Append([]byte("["), []byte("abc")) | ||
| if err != nil || string(got) != `["YWJj"` { | ||
| t.Fatalf("got %q, %v", got, err) | ||
| } | ||
| } | ||
|
|
||
| // The parity tests would still pass if the fast path were silently disabled | ||
| // (json.Marshal is the fallback), so pin that these never reach it. | ||
| func TestAppendFastPathDoesNotAllocate(t *testing.T) { | ||
| buf := make([]byte, 0, 64) | ||
| for _, v := range []any{int64(-9000), int32(42), uint64(7), 3.14, 0.30000000000000004, 1e-7, float32(0.5), true, | ||
| "alpha", "café 日本語 😀"} { | ||
| if n := testing.AllocsPerRun(100, func() { buf, _ = Append(buf[:0], v) }); n != 0 { | ||
| t.Errorf("Append(%#v) allocated %v times per run, want 0", v, n) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // jsonfmt appends into a reused buffer, as both renderers do. | ||
| func BenchmarkAppendScalars(b *testing.B) { | ||
| // The last three take json.Marshal's fallback. | ||
| values := []any{int32(42), int64(-9000), 3.14, 0.30000000000000004, float32(0.5), true, "alpha0", "café crème", | ||
| `a"b`, []byte("abc"), math.NaN()} | ||
| b.Run("jsonfmt", func(b *testing.B) { | ||
| b.ReportAllocs() | ||
| buf := make([]byte, 0, 64) | ||
| for i := 0; i < b.N; i++ { | ||
| for _, v := range values { | ||
| buf, _ = Append(buf[:0], v) | ||
| } | ||
| } | ||
| }) | ||
| b.Run("encoding_json", func(b *testing.B) { | ||
| b.ReportAllocs() | ||
| for i := 0; i < b.N; i++ { | ||
| for _, v := range values { | ||
| _, _ = json.Marshal(v) | ||
| } | ||
| } | ||
| }) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔵 Low — Together these exhaustive and randomized parity tests make about 2M
json.Marshalcalls: every rune up toutf8.MaxRune(~1.1M), 100k random strings, and 800k floats. None of them checkstesting.Short(), so they all run undergo test -short ./...and the-raceCI job. Under the race detector that can add many seconds to every run. Consider skipping the exhaustive rune sweep and cutting the random iteration counts whentesting.Short()is set, and keeping the full sweep for non-short runs.