From f00970493d68811b03fc73b5de7979084337fd1c Mon Sep 17 00:00:00 2001
From: Ashlie Martinez <ashmrtnz@alcion.ai>
Date: Thu, 30 Mar 2023 10:40:08 -0700
Subject: [PATCH] Helper programs to repro JSON deserialize mem use

JSON deserialize of arrays is inefficient for memory. This is a minimal
reproducer to show that it uses lots more memory than expected.

Build each program separately, run gen and then you can run goread for
the repro and some memory usage stats.
---
 src/cmd/jsondebug/common/helpers.go | 31 +++++++++++++++++++
 src/cmd/jsondebug/gen/gen.go        | 47 +++++++++++++++++++++++++++++
 src/cmd/jsondebug/goread/read.go    | 30 ++++++++++++++++++
 3 files changed, 108 insertions(+)
 create mode 100644 src/cmd/jsondebug/common/helpers.go
 create mode 100644 src/cmd/jsondebug/gen/gen.go
 create mode 100644 src/cmd/jsondebug/goread/read.go

diff --git a/src/cmd/jsondebug/common/helpers.go b/src/cmd/jsondebug/common/helpers.go
new file mode 100644
index 000000000..6213cbde7
--- /dev/null
+++ b/src/cmd/jsondebug/common/helpers.go
@@ -0,0 +1,31 @@
+package common
+
+import (
+	"fmt"
+	"runtime"
+)
+
+const (
+	NumItems = 300000
+	ItemSize = 1024
+	FileName = "input.json"
+)
+
+type Foo struct {
+	A []byte
+}
+
+func PrintMemUsage() {
+	var m runtime.MemStats
+
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
+	fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
+	fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
+	fmt.Printf("\tNumGC = %v\n", m.NumGC)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}
diff --git a/src/cmd/jsondebug/gen/gen.go b/src/cmd/jsondebug/gen/gen.go
new file mode 100644
index 000000000..6ce593f11
--- /dev/null
+++ b/src/cmd/jsondebug/gen/gen.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/alcionai/corso/src/cmd/jsondebug/common"
+)
+
+func main() {
+	buf := make([]byte, common.ItemSize)
+	data := make([]common.Foo, 0, common.NumItems)
+
+	for i := 0; i < common.NumItems; i++ {
+		n, err := rand.Read(buf)
+		if err != nil {
+			fmt.Printf("Error reading random data: %v\n", err)
+			return
+		} else if n != common.ItemSize {
+			fmt.Printf(
+				"Short read for item data: wanted %d, got %d\n",
+				common.ItemSize,
+				n,
+			)
+			return
+		}
+
+		item := common.Foo{A: buf}
+		data = append(data, item)
+	}
+
+	f, err := os.Create(common.FileName)
+	if err != nil {
+		fmt.Printf("Error making output file: %v\n", err)
+		return
+	}
+
+	defer f.Close()
+
+	enc := json.NewEncoder(f)
+	if err := enc.Encode(data); err != nil {
+		fmt.Printf("Error writing json to file: %v\n", err)
+		return
+	}
+}
diff --git a/src/cmd/jsondebug/goread/read.go b/src/cmd/jsondebug/goread/read.go
new file mode 100644
index 000000000..39ba3e006
--- /dev/null
+++ b/src/cmd/jsondebug/goread/read.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/alcionai/corso/src/cmd/jsondebug/common"
+)
+
+func main() {
+	f, err := os.Open(common.FileName)
+	if err != nil {
+		fmt.Printf("Error opening input file: %v\n", err)
+		return
+	}
+
+	defer f.Close()
+
+	dec := json.NewDecoder(f)
+
+	output := []common.Foo{}
+
+	if err := dec.Decode(&output); err != nil {
+		fmt.Printf("Error decoding input: %v\n", err)
+		return
+	}
+
+	common.PrintMemUsage()
+}