make gomemlimits 500mb

src/corso.go

@@ -1,9 +1,102 @@
 package main
 
 import (
+	"context"
+	"log"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"runtime/pprof"
+	"strconv"
+	"time"
+
 	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/pkg/profile"
 )
 
+var profileTicker = time.NewTicker(1 * time.Second)
+var perMinuteMap = make(map[time.Time]int)
+var timeSinceRefresh = time.Now()
+
+//var profileTicker = time.NewTicker(120 * time.Second)
+
+var printTicker = time.NewTicker(1 * time.Second)
+var profileCounter = 0
+
 func main() {
+	defer profile.Start(profile.MemProfile).Stop()
+	debug.SetMemoryLimit(0.5 * 1024 * 1024 * 1024)
+
+	go func() {
+		for {
+			select {
+			case <-profileTicker.C:
+				var m runtime.MemStats
+				runtime.ReadMemStats(&m)
+
+				// if mem > 3GB and we havent captured a profile this min, capture it
+				// or if its been 2 mins since last profile, capture it
+				t := time.Now().Truncate(time.Minute)
+				//if (m.HeapAlloc > uint64(3*1024*1024*1024) && perMinuteMap[t] == 0) || time.Since(timeSinceRefresh) > 2*time.Minute {
+				if time.Since(timeSinceRefresh) > 2*time.Minute {
+					filename := "mem." + strconv.Itoa(profileCounter) + ".pprof"
+
+					f, _ := os.Create(filename)
+					if err := pprof.WriteHeapProfile(f); err != nil {
+						log.Fatal("could not write memory profile: ", err)
+					}
+
+					f.Close()
+
+					profileCounter++
+					perMinuteMap[t] = 1
+					timeSinceRefresh = time.Now()
+				}
+
+			}
+		}
+	}()
+
+	go func() {
+		for {
+			select {
+			case <-printTicker.C:
+				PrintMemUsage()
+			}
+		}
+	}()
+
 	cli.Handle()
 }
+
+// PrintMemUsage outputs the current, total and OS memory being used. As well as the number
+// of garage collection cycles completed.
+func PrintMemUsage() {
+	ctx := context.Background()
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	// logger.Ctx(ctx).Info("Alloc = ", bToMb(m.Alloc), " MB")
+	// logger.Ctx(ctx).Info("TotalAlloc = ", bToMb(m.TotalAlloc), " MB")
+	logger.Ctx(ctx).Info("HeapAlloc = ", bToMb(m.HeapAlloc), " MB") // same as Alloc
+
+	logger.Ctx(ctx).Info("HeapReleased = ", bToMb(m.HeapReleased), " MB")
+	logger.Ctx(ctx).Info("HeapObjects = ", bToMb(m.HeapObjects), " MB")
+	logger.Ctx(ctx).Info("HeapSys = ", bToMb(m.HeapSys), " MB")
+	logger.Ctx(ctx).Info("HeapIdle = ", bToMb(m.HeapIdle), " MB")
+	logger.Ctx(ctx).Info("HeapInuse = ", bToMb(m.HeapInuse), " MB")
+
+	// logger.Ctx(ctx).Info("Mallocs = ", bToMb(m.Mallocs), " MB")
+	// logger.Ctx(ctx).Info("Frees = ", bToMb(m.Frees), " MB")
+
+	// logger.Ctx(ctx).Info("StackInuse = ", bToMb(m.StackInuse), " MB")
+	// logger.Ctx(ctx).Info("StackSys = ", bToMb(m.StackSys), " MB")
+
+	// logger.Ctx(ctx).Info("Sys = ", bToMb(m.Sys), " MB")
+	logger.Ctx(ctx).Info("NumGC = ", m.NumGC)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}

src/go.mod

@@ -3,6 +3,8 @@ module github.com/alcionai/corso/src
 go 1.21
 
 replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230822191057-17d4deff94a3
+//replace github.com/DmitriyVTitov/size => /Users/pandeyab/temp/size
+
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1
@@ -46,15 +48,18 @@ require (
 )
 
 require (
+	github.com/DmitriyVTitov/size v1.5.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/aws/aws-sdk-go v1.47.9 // indirect
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/cronexpr v1.1.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -64,6 +69,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pkg/profile v1.7.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect

src/go.sum

@@ -49,6 +49,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM=
 github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
+github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI=
 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
@@ -121,6 +123,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -155,6 +159,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -207,6 +212,9 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk=
+github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
@@ -241,6 +249,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
@@ -353,6 +362,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
@@ -631,6 +642,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

src/internal/m365/collection/drive/collection.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/spatialcurrent/go-lazy/pkg/lazy"
 
 	"github.com/alcionai/corso/src/internal/common/idname"
@@ -52,7 +51,7 @@ type Collection struct {
 	// represents
 	folderPath path.Path
 	// M365 IDs of file items within this collection
-	driveItems map[string]models.DriveItemable
+	driveItems map[string]CorsoDriveItemable
 
 	// Primary M365 ID of the drive this collection was created from
 	driveID   string
@@ -92,6 +91,10 @@ type Collection struct {
 	counter *count.Bus
 }
 
+func (c *Collection) GetDriveItemsMap() map[string]CorsoDriveItemable {
+	return c.driveItems
+}
+
 func pathToLocation(p path.Path) (*path.Builder, error) {
 	if p == nil {
 		return nil, nil
@@ -172,7 +175,7 @@ func newColl(
 		protectedResource:         resource,
 		folderPath:                currPath,
 		prevPath:                  prevPath,
-		driveItems:                map[string]models.DriveItemable{},
+		driveItems:                map[string]CorsoDriveItemable{},
 		driveID:                   driveID,
 		data:                      dataCh,
 		statusUpdater:             statusUpdater,
@@ -190,9 +193,13 @@ func newColl(
 // Adds an itemID to the collection.  This will make it eligible to be
 // populated. The return values denotes if the item was previously
 // present or is new one.
-func (oc *Collection) Add(item models.DriveItemable) bool {
+func (oc *Collection) Add(cdi CorsoDriveItemable) bool {
-	_, found := oc.driveItems[ptr.Val(item.GetId())]
+	// _, found := oc.driveItems[ptr.Val(item.GetId())]
-	oc.driveItems[ptr.Val(item.GetId())] = item
+	// oc.driveItems[ptr.Val(item.GetId())] = item
+
+	//cdi := ToCorsoDriveItemable(item)
+	_, found := oc.driveItems[ptr.Val(cdi.GetId())]
+	oc.driveItems[ptr.Val(cdi.GetId())] = cdi
 
 	// if !found, it's a new addition
 	return !found
@@ -261,13 +268,13 @@ func (oc Collection) DoNotMergeItems() bool {
 func (oc *Collection) getDriveItemContent(
 	ctx context.Context,
 	driveID string,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	errs *fault.Bus,
 ) (io.ReadCloser, error) {
-	var (
+	// var (
-		itemID   = ptr.Val(item.GetId())
+	// 	itemID   = ptr.Val(item.GetId())
-		itemName = ptr.Val(item.GetName())
+	// 	itemName = ptr.Val(item.GetName())
-	)
+	// )
 
 	itemData, err := downloadContent(
 		ctx,
@@ -279,7 +286,7 @@ func (oc *Collection) getDriveItemContent(
 	if err != nil {
 		if clues.HasLabel(err, graph.LabelsMalware) || (item != nil && item.GetMalware() != nil) {
 			logger.CtxErr(ctx, err).With("skipped_reason", fault.SkipMalware).Info("item flagged as malware")
-			errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
+			//errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
 
 			return nil, clues.Wrap(err, "malware item").Label(graph.LabelsSkippable)
 		}
@@ -309,12 +316,12 @@ func (oc *Collection) getDriveItemContent(
 				CtxErr(ctx, err).
 				With("skipped_reason", fault.SkipOneNote).
 				Info("inaccessible one note file")
-			errs.AddSkip(ctx, fault.FileSkip(
+			// errs.AddSkip(ctx, fault.FileSkip(
-				fault.SkipOneNote,
+			// 	fault.SkipOneNote,
-				driveID,
+			// 	driveID,
-				itemID,
+			// 	itemID,
-				itemName,
+			// 	itemName,
-				graph.ItemInfo(item)))
+			// 	graph.ItemInfo(item)))
 
 			return nil, clues.Wrap(err, "inaccesible oneNote item").Label(graph.LabelsSkippable)
 		}
@@ -344,7 +351,7 @@ func downloadContent(
 	ctx context.Context,
 	iaag itemAndAPIGetter,
 	uc getItemPropertyer,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	driveID string,
 	counter *count.Bus,
 ) (io.ReadCloser, error) {
@@ -379,7 +386,9 @@ func downloadContent(
 		return nil, clues.Wrap(err, "retrieving expired item")
 	}
 
-	content, err = downloadItem(ctx, iaag, di)
+	cdi := ToCorsoDriveItemable(di)
+
+	content, err = downloadItem(ctx, iaag, cdi)
 	if err != nil {
 		return nil, clues.Wrap(err, "content download retry")
 	}
@@ -473,7 +482,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 
 		wg.Add(1)
 
-		go func(item models.DriveItemable) {
+		go func(item CorsoDriveItemable) {
 			defer wg.Done()
 			defer func() { <-semaphoreCh }()
 
@@ -497,14 +506,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 
 type lazyItemGetter struct {
 	info                 *details.ItemInfo
-	item                 models.DriveItemable
+	item                 CorsoDriveItemable
 	driveID              string
 	suffix               string
 	itemExtensionFactory []extensions.CreateItemExtensioner
 	contentGetter        func(
 		ctx context.Context,
 		driveID string,
-		item models.DriveItemable,
+		item CorsoDriveItemable,
 		errs *fault.Bus) (io.ReadCloser, error)
 }
 
@@ -545,7 +554,7 @@ func (lig *lazyItemGetter) GetData(
 func (oc *Collection) streamDriveItem(
 	ctx context.Context,
 	parentPath *path.Builder,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	stats *driveStats,
 	itemExtensionFactory []extensions.CreateItemExtensioner,
 	errs *fault.Bus,

src/internal/m365/collection/drive/collections.go

@@ -696,7 +696,7 @@ func (c *Collections) handleDelete(
 
 func (c *Collections) getCollectionPath(
 	driveID string,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 ) (path.Path, error) {
 	var (
 		pb     = odConsts.DriveFolderPrefixBuilder(driveID)
@@ -845,7 +845,7 @@ func (c *Collections) PopulateDriveCollections(
 
 func (c *Collections) processItem(
 	ctx context.Context,
-	item models.DriveItemable,
+	di models.DriveItemable,
 	driveID, driveName string,
 	oldPrevPaths, currPrevPaths, newPrevPaths map[string]string,
 	seenFolders map[string]string,
@@ -855,6 +855,8 @@ func (c *Collections) processItem(
 	counter *count.Bus,
 	skipper fault.AddSkipper,
 ) error {
+	item := ToCorsoDriveItemable(di)
+
 	var (
 		itemID   = ptr.Val(item.GetId())
 		itemName = ptr.Val(item.GetName())
@@ -868,7 +870,7 @@ func (c *Collections) processItem(
 		"item_is_folder", isFolder)
 
 	if item.GetMalware() != nil {
-		addtl := graph.ItemInfo(item)
+		addtl := graph.ItemInfo(di)
 		skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
 
 		if isFolder {

src/internal/m365/collection/drive/custom_drive_item.go

@@ -0,0 +1,285 @@
+package drive
+
+import (
+	"time"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+)
+
+// Replica of models.DriveItemable
+type CorsoDriveItemable interface {
+	GetId() *string
+	GetName() *string
+	GetSize() *int64
+	GetFile() fileDriveItemable
+	GetFolder() folderDriveItemable
+	GetPackageEscaped() packageDriveItemable
+	GetParentReference() parentReferenceable
+	GetAdditionalData() map[string]interface{}
+	SetParentReference(parentReferenceable)
+	GetShared() itemSharedable
+	GetCreatedBy() itemIdentitySetable
+	GetCreatedDateTime() *time.Time
+	GetLastModifiedDateTime() *time.Time
+	GetMalware() malwareable
+	GetDeleted() deletedable
+	GetRoot() itemRootable
+	// Not used anywhere
+	//GetSharepointIds() sharepointIdsable
+}
+
+type fileDriveItemable interface {
+	GetMimeType() *string
+}
+type folderDriveItemable interface{}
+type packageDriveItemable interface{}
+type parentReferenceable interface {
+	GetPath() *string
+	GetId() *string
+	GetName() *string
+	GetDriveId() *string
+}
+type itemSharedable interface{}
+type malwareable interface{}
+type deletedable interface{}
+type itemRootable interface{}
+type itemIdentitySetable interface {
+	GetUser() itemUserable
+}
+type itemUserable interface {
+	GetAdditionalData() map[string]interface{}
+}
+
+// Concrete implementations
+type folderDriveItem struct {
+	isFolder bool
+}
+
+type fileDriveItem struct {
+	isFile   bool
+	mimeType *string
+}
+
+func (fdi *fileDriveItem) GetMimeType() *string {
+	return fdi.mimeType
+}
+
+type packageDriveItem struct {
+	isPackage bool
+}
+
+type parentReference struct {
+	path    *string
+	id      *string
+	name    *string
+	driveId *string
+}
+
+func (pr *parentReference) GetPath() *string {
+	return pr.path
+}
+
+func (pr *parentReference) GetId() *string {
+	return pr.id
+}
+
+func (pr *parentReference) GetName() *string {
+	return pr.name
+}
+
+func (pr *parentReference) GetDriveId() *string {
+	return pr.driveId
+}
+
+type itemShared struct {
+	isShared bool
+}
+
+type itemMalware struct {
+	isMalware bool
+}
+
+type itemDeleted struct {
+	isDeleted bool
+}
+
+type itemRoot struct {
+	isRoot bool
+}
+
+type itemIdentitySet struct {
+	user itemUserable
+}
+
+func (iis *itemIdentitySet) GetUser() itemUserable {
+	return iis.user
+}
+
+type itemUser struct {
+	additionalData map[string]interface{}
+}
+
+func (iu *itemUser) GetAdditionalData() map[string]interface{} {
+	return iu.additionalData
+}
+
+type CorsoDriveItem struct {
+	ID                   string
+	Name                 string
+	Size                 int64
+	File                 fileDriveItemable
+	Folder               folderDriveItemable
+	Package              packageDriveItemable
+	AdditionalData       map[string]interface{}
+	ParentReference      parentReferenceable
+	Shared               itemSharedable
+	CreatedBy            itemIdentitySetable
+	CreatedDateTime      *time.Time
+	LastModifiedDateTime *time.Time
+	Malware              malwareable
+	Deleted              deletedable
+	Root                 itemRootable
+}
+
+func (c *CorsoDriveItem) GetId() *string {
+	return &c.ID
+}
+
+func (c *CorsoDriveItem) GetName() *string {
+	return &c.Name
+}
+
+func (c *CorsoDriveItem) GetSize() *int64 {
+	return &c.Size
+}
+
+func (c *CorsoDriveItem) GetFile() fileDriveItemable {
+	return c.File
+}
+
+func (c *CorsoDriveItem) GetFolder() folderDriveItemable {
+	return c.Folder
+}
+
+func (c *CorsoDriveItem) GetPackageEscaped() packageDriveItemable {
+	return c.Package
+}
+
+func (c *CorsoDriveItem) GetParentReference() parentReferenceable {
+	return c.ParentReference
+}
+
+func (c *CorsoDriveItem) SetParentReference(parent parentReferenceable) {
+	c.ParentReference = parent
+}
+
+func (c *CorsoDriveItem) GetAdditionalData() map[string]interface{} {
+	return c.AdditionalData
+}
+
+func (c *CorsoDriveItem) GetShared() itemSharedable {
+	return c.Shared
+}
+
+func (c *CorsoDriveItem) GetCreatedBy() itemIdentitySetable {
+	return c.CreatedBy
+}
+
+func (c *CorsoDriveItem) GetCreatedDateTime() *time.Time {
+	return c.CreatedDateTime
+}
+
+func (c *CorsoDriveItem) GetLastModifiedDateTime() *time.Time {
+	return c.LastModifiedDateTime
+}
+
+func (c *CorsoDriveItem) GetMalware() malwareable {
+	return c.Malware
+}
+
+func (c *CorsoDriveItem) GetDeleted() deletedable {
+	return c.Deleted
+}
+
+func (c *CorsoDriveItem) GetRoot() itemRootable {
+	return c.Root
+}
+
+// func (c *CorsoDriveItem) GetSharepointIds() sharepointIdsable {
+// 	return nil
+// }
+
+// models.DriveItemable to CorsoDriveItemable
+func ToCorsoDriveItemable(item models.DriveItemable) CorsoDriveItemable {
+	cdi := &CorsoDriveItem{
+		ID:                   ptr.Val(item.GetId()),
+		Name:                 ptr.Val(item.GetName()),
+		Size:                 ptr.Val(item.GetSize()),
+		CreatedDateTime:      item.GetCreatedDateTime(),
+		LastModifiedDateTime: item.GetLastModifiedDateTime(),
+		AdditionalData:       item.GetAdditionalData(),
+	}
+
+	if item.GetFolder() != nil {
+		cdi.Folder = &folderDriveItem{
+			isFolder: true,
+		}
+	}
+
+	if item.GetFile() != nil {
+		cdi.File = &fileDriveItem{
+			isFile:   true,
+			mimeType: item.GetFile().GetMimeType(),
+		}
+	}
+
+	if item.GetPackageEscaped() != nil {
+		cdi.Package = &packageDriveItem{
+			isPackage: true,
+		}
+	}
+
+	if item.GetParentReference() != nil {
+		cdi.ParentReference = &parentReference{
+			id:      item.GetParentReference().GetId(),
+			path:    item.GetParentReference().GetPath(),
+			name:    item.GetParentReference().GetName(),
+			driveId: item.GetParentReference().GetDriveId(),
+		}
+	}
+
+	if item.GetShared() != nil {
+		cdi.Shared = &itemShared{
+			isShared: true,
+		}
+	}
+
+	if item.GetMalware() != nil {
+		cdi.Malware = &itemMalware{
+			isMalware: true,
+		}
+	}
+
+	if item.GetDeleted() != nil {
+		cdi.Deleted = &itemDeleted{
+			isDeleted: true,
+		}
+	}
+
+	if item.GetRoot() != nil {
+		cdi.Root = &itemRoot{
+			isRoot: true,
+		}
+	}
+
+	if item.GetCreatedBy() != nil && item.GetCreatedBy().GetUser() != nil {
+		cdi.CreatedBy = &itemIdentitySet{
+			user: &itemUser{
+				additionalData: item.GetCreatedBy().GetUser().GetAdditionalData(),
+			},
+		}
+	}
+
+	return cdi
+}

src/internal/m365/collection/drive/group_handler.go

@@ -2,7 +2,6 @@ package drive
 
 import (
 	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/common/ptr"
@@ -105,7 +104,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) {
 func (h groupBackupHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/m365/collection/drive/handler_utils.go

@@ -3,12 +3,10 @@ package drive
 import (
 	"strings"
 
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
-
 	"github.com/alcionai/corso/src/internal/common/ptr"
 )
 
-func getItemCreator(item models.DriveItemable) string {
+func getItemCreator(item CorsoDriveItemable) string {
 	if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil {
 		return ""
 	}
@@ -30,7 +28,7 @@ func getItemCreator(item models.DriveItemable) string {
 	return *ed.(*string)
 }
 
-func getItemDriveInfo(item models.DriveItemable) (string, string) {
+func getItemDriveInfo(item CorsoDriveItemable) (string, string) {
 	if item.GetParentReference() == nil {
 		return "", ""
 	}

src/internal/m365/collection/drive/handlers.go

@@ -23,7 +23,7 @@ type ItemInfoAugmenter interface {
 	AugmentItemInfo(
 		dii details.ItemInfo,
 		resource idname.Provider,
-		item models.DriveItemable,
+		item CorsoDriveItemable,
 		size int64,
 		parentPath *path.Builder,
 	) details.ItemInfo

src/internal/m365/collection/drive/item.go

@@ -34,7 +34,7 @@ var downloadURLKeys = []string{
 func downloadItem(
 	ctx context.Context,
 	ag api.Getter,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 ) (io.ReadCloser, error) {
 	if item == nil {
 		return nil, clues.New("nil item")
@@ -152,7 +152,7 @@ func downloadItemMeta(
 	ctx context.Context,
 	getter GetItemPermissioner,
 	driveID string,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 ) (io.ReadCloser, int, error) {
 	meta := metadata.Metadata{
 		FileName:    ptr.Val(item.GetName()),
@@ -205,12 +205,16 @@ func driveItemWriter(
 	return iw, ptr.Val(icu.GetUploadUrl()), nil
 }
 
-func setName(orig models.ItemReferenceable, driveName string) models.ItemReferenceable {
+func setName(orig parentReferenceable, driveName string) models.ItemReferenceable {
 	if orig == nil {
 		return nil
 	}
 
-	orig.SetName(&driveName)
+	mod := models.NewItemReference()
+	mod.SetDriveId(orig.GetDriveId())
+	mod.SetId(orig.GetId())
+	mod.SetPath(orig.GetPath())
+	mod.SetName(&driveName)
 
-	return orig
+	return mod
 }

src/internal/m365/collection/drive/restore.go

@@ -863,7 +863,7 @@ func restoreFile(
 	dii := ir.AugmentItemInfo(
 		details.ItemInfo{},
 		rcc.ProtectedResource,
-		newItem,
+		nil,
 		written,
 		nil)
 

src/internal/m365/collection/drive/site_handler.go

@@ -33,7 +33,7 @@ func (h baseSiteHandler) NewDrivePager(
 func (h baseSiteHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/m365/collection/drive/user_drive_handler.go

@@ -42,7 +42,7 @@ func (h baseUserDriveHandler) NewDrivePager(
 func (h baseUserDriveHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/operations/backup.go

@@ -16,6 +16,7 @@ import (
 	"github.com/alcionai/corso/src/internal/events"
 	"github.com/alcionai/corso/src/internal/kopia"
 	kinject "github.com/alcionai/corso/src/internal/kopia/inject"
+	"github.com/alcionai/corso/src/internal/m365/collection/drive"
 	"github.com/alcionai/corso/src/internal/model"
 	"github.com/alcionai/corso/src/internal/observe"
 	"github.com/alcionai/corso/src/internal/operations/inject"
@@ -439,67 +440,111 @@ func (op *BackupOperation) do(
 		lastBackupVersion = mans.MinBackupVersion()
 	}
 
-	// TODO(ashmrtn): This should probably just return a collection that deletes
+	iterations := 1
-	// the entire subtree instead of returning an additional bool. That way base
+
-	// selection is controlled completely by flags and merging is controlled
+	for i := 0; i < iterations; i++ {
-	// completely by collections.
+		// TODO(ashmrtn): This should probably just return a collection that deletes
-	cs, ssmb, canUsePreviousBackup, err := produceBackupDataCollections(
+		// the entire subtree instead of returning an additional bool. That way base
-		ctx,
+		// selection is controlled completely by flags and merging is controlled
-		op.bp,
+		// completely by collections.
-		op.ResourceOwner,
+		cs, _, _, err := produceBackupDataCollections(
-		op.Selectors,
+			ctx,
-		mdColls,
+			op.bp,
-		lastBackupVersion,
+			op.ResourceOwner,
-		op.Options,
+			op.Selectors,
-		op.Counter,
+			mdColls,
-		op.Errors)
+			lastBackupVersion,
-	if err != nil {
+			op.Options,
-		return nil, clues.Wrap(err, "producing backup data collections")
+			op.Counter,
+			op.Errors)
+		if err != nil {
+			return nil, clues.Wrap(err, "producing backup data collections")
+		}
+
+		// Sleep for 4 mins to let the memory usage settle down so that we have a better
+		// picture. Also allows pprof to run twice during this time.
+		// Do some meaningless work after to make sure the collections dont get garbage collected
+		time.Sleep(4 * time.Minute)
+
+		// sum := 0
+		// numItems := 0
+		// mapSum := 0
+
+		for _, c := range cs {
+			v, ok := c.(*drive.Collection)
+			if !ok {
+				continue
+			}
+
+			m := v.GetDriveItemsMap()
+			for key := range m {
+				logger.Ctx(ctx).Debug(key)
+			}
+
+			// Get sizeof recursively using reflect
+			// m := v.GetDriveItemsMap()
+			// for _, val := range m {
+			// 	s := size.Of(val)
+			// 	sum += s
+			// 	numItems++
+			// }
+
+			// ms := size.Of(m)
+			// mapSum += ms
+
+			// logger.Ctx(ctx).Debugf("coll drive map size %d, num drive items %d\n", ms, len(m))
+		}
+
+		// print total sum
+		// logger.Ctx(ctx).Debugf("itemSum %d, map sum %d, total items %d, mem used per item %f mem per item in map %f \n", sum, mapSum, numItems, float64(sum)/float64(numItems), float64(mapSum)/float64(numItems))
 	}
 
-	ctx = clues.Add(
+	return nil, clues.New("failed")
-		ctx,
-		"can_use_previous_backup", canUsePreviousBackup,
-		"collection_count", len(cs))
 
-	writeStats, deets, toMerge, err := consumeBackupCollections(
+	// ctx = clues.Add(
-		ctx,
+	// 	ctx,
-		op.kopia,
+	// 	"can_use_previous_backup", canUsePreviousBackup,
-		op.account.ID(),
+	// 	"collection_count", len(cs))
-		reasons,
-		mans,
-		cs,
-		ssmb,
-		backupID,
-		op.incremental && canUseMetadata && canUsePreviousBackup,
-		op.Counter,
-		op.Errors)
-	if err != nil {
-		return nil, clues.Wrap(err, "persisting collection backups")
-	}
 
-	opStats.hasNewDetailEntries = (deets != nil && !deets.Empty()) ||
+	// writeStats, deets, toMerge, err := consumeBackupCollections(
-		(toMerge != nil && toMerge.ItemsToMerge() > 0)
+	// 	ctx,
-	opStats.k = writeStats
+	// 	op.kopia,
+	// 	op.account.ID(),
+	// 	reasons,
+	// 	mans,
+	// 	cs,
+	// 	ssmb,
+	// 	backupID,
+	// 	op.incremental && canUseMetadata && canUsePreviousBackup,
+	// 	op.Counter,
+	// 	op.Errors)
+	// if err != nil {
+	// 	return nil, clues.Wrap(err, "persisting collection backups")
+	// }
 
-	err = mergeDetails(
+	// opStats.hasNewDetailEntries = (deets != nil && !deets.Empty()) ||
-		ctx,
+	// 	(toMerge != nil && toMerge.ItemsToMerge() > 0)
-		detailsStore,
+	// opStats.k = writeStats
-		mans,
-		toMerge,
-		deets,
-		writeStats,
-		op.Selectors.PathService(),
-		op.Errors)
-	if err != nil {
-		return nil, clues.Wrap(err, "merging details")
-	}
 
-	opStats.ctrl = op.bp.Wait()
+	// err = mergeDetails(
+	// 	ctx,
+	// 	detailsStore,
+	// 	mans,
+	// 	toMerge,
+	// 	deets,
+	// 	writeStats,
+	// 	op.Selectors.PathService(),
+	// 	op.Errors)
+	// if err != nil {
+	// 	return nil, clues.Wrap(err, "merging details")
+	// }
 
-	logger.Ctx(ctx).Debug(opStats.ctrl)
+	// opStats.ctrl = op.bp.Wait()
+
+	// logger.Ctx(ctx).Debug(opStats.ctrl)
+
+	// return deets, nil
 
-	return deets, nil
 }
 
 func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) {