Jsonparser prototype

JSON deserialize of arrays is inefficient for memory. This is a minimal
reproducer to show that it uses lots more memory than expected.

Build each program separately, run gen and then you can run goread for
the repro and some memory usage stats.

src/cmd/jsondebug/common/helpers.go

@@ -0,0 +1,35 @@
+package common
+
+import (
+	"fmt"
+	"runtime"
+)
+
+const (
+	NumItems = 300000
+	ItemSize = 1024
+	FileName = "input.json"
+)
+
+type FooArray struct {
+	Entries []*Foo `json:"entries"`
+}
+
+type Foo struct {
+	A []byte
+}
+
+func PrintMemUsage() {
+	var m runtime.MemStats
+
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
+	fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
+	fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
+	fmt.Printf("\tNumGC = %v\n", m.NumGC)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}

src/cmd/jsondebug/gen/gen.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/alcionai/corso/src/cmd/jsondebug/common"
+)
+
+func main() {
+	buf := make([]byte, common.ItemSize)
+	data := &common.FooArray{
+		Entries: make([]*common.Foo, 0, common.NumItems),
+	}
+
+	for i := 0; i < common.NumItems; i++ {
+		n, err := rand.Read(buf)
+		if err != nil {
+			fmt.Printf("Error reading random data: %v\n", err)
+			return
+		} else if n != common.ItemSize {
+			fmt.Printf(
+				"Short read for item data: wanted %d, got %d\n",
+				common.ItemSize,
+				n,
+			)
+			return
+		}
+
+		item := common.Foo{A: buf}
+		data.Entries = append(data.Entries, &item)
+	}
+
+	f, err := os.Create(common.FileName)
+	if err != nil {
+		fmt.Printf("Error making output file: %v\n", err)
+		return
+	}
+
+	defer f.Close()
+
+	enc := json.NewEncoder(f)
+	if err := enc.Encode(data); err != nil {
+		fmt.Printf("Error writing json to file: %v\n", err)
+		return
+	}
+}

src/cmd/jsondebug/goread/read.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/alcionai/corso/src/cmd/jsondebug/common"
+)
+
+func main() {
+	f, err := os.Open(common.FileName)
+	if err != nil {
+		fmt.Printf("Error opening input file: %v\n", err)
+		return
+	}
+
+	defer f.Close()
+
+	dec := json.NewDecoder(f)
+
+	output := common.FooArray{}
+
+	if err := dec.Decode(&output); err != nil {
+		fmt.Printf("Error decoding input: %v\n", err)
+		return
+	}
+
+	common.PrintMemUsage()
+}

src/cmd/jsondebug/jsonparser/read.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/alcionai/corso/src/cmd/jsondebug/common"
+	"github.com/buger/jsonparser"
+)
+
+func main() {
+	f, err := os.Open(common.FileName)
+	if err != nil {
+		fmt.Printf("Error opening input file: %v\n", err)
+		return
+	}
+
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		fmt.Printf("Error reading file: %v\n", err)
+		return
+	}
+
+	output := common.FooArray{
+		Entries: []*common.Foo{},
+	}
+
+	_ = output
+
+	common.PrintMemUsage()
+
+	jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+		buf, _, _, errInner := jsonparser.Get(value, "A")
+		if errInner != nil {
+			fmt.Printf("Error decoding input: %v\n", err)
+			return
+		}
+		_ = value
+		cpBuf := make([]byte, len(buf))
+		_ = copy(cpBuf, buf)
+		e := &common.Foo{
+			A: cpBuf,
+		}
+		output.Entries = append(output.Entries, e)
+	}, "entries")
+
+	common.PrintMemUsage()
+}

src/go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/dnaeon/go-vcr v1.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect

src/go.sum

@@ -66,6 +66,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
 github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=