Some misc changes

src/corso.go

@@ -1,9 +1,105 @@
 package main
 
 import (
+	"context"
+	"log"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"runtime/pprof"
+	"strconv"
+	"time"
+
+	"github.com/pkg/profile"
+
 	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/pkg/logger"
+)
+
+var (
+	profileTicker    = time.NewTicker(1 * time.Second)
+	perMinuteMap     = make(map[time.Time]int)
+	timeSinceRefresh = time.Now()
+)
+
+// var profileTicker = time.NewTicker(120 * time.Second)
+var (
+	printTicker    = time.NewTicker(1 * time.Second)
+	profileCounter = 0
 )
 
 func main() {
+	defer profile.Start(profile.MemProfile).Stop()
+	debug.SetMemoryLimit(1 * 1024 * 1024 * 1024)
+
+	go func() {
+		for {
+			select {
+			case <-profileTicker.C:
+				var m runtime.MemStats
+				runtime.ReadMemStats(&m)
+
+				// if mem > 3GB and we havent captured a profile this min, capture it
+				// or if its been 2 mins since last profile, capture it
+				t := time.Now().Truncate(time.Minute)
+				// if (m.HeapAlloc > uint64(3*1024*1024*1024) && perMinuteMap[t] == 0) || time.Since(timeSinceRefresh) > 2*time.Minute {
+				if time.Since(timeSinceRefresh) > 2*time.Minute {
+					filename := "mem." + strconv.Itoa(profileCounter) + ".pprof"
+
+					f, _ := os.Create(filename)
+					if err := pprof.WriteHeapProfile(f); err != nil {
+						log.Fatal("could not write memory profile: ", err)
+					}
+
+					f.Close()
+
+					profileCounter++
+					perMinuteMap[t] = 1
+					timeSinceRefresh = time.Now()
+				}
+			}
+		}
+	}()
+
+	go func() {
+		for {
+			select {
+			case <-printTicker.C:
+				PrintMemUsage()
+			}
+		}
+	}()
+
 	cli.Handle()
 }
+
+// PrintMemUsage outputs the current, total and OS memory being used. As well as the number
+// of garage collection cycles completed.
+func PrintMemUsage() {
+	ctx := context.Background()
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	// logger.Ctx(ctx).Info("Alloc = ", bToMb(m.Alloc), " MB")
+	// logger.Ctx(ctx).Info("TotalAlloc = ", bToMb(m.TotalAlloc), " MB")
+	logger.Ctx(ctx).Info("HeapAlloc = ", bToMb(m.HeapAlloc), " MB") // same as Alloc
+
+	logger.Ctx(ctx).Info("HeapReleased = ", bToMb(m.HeapReleased), " MB")
+	logger.Ctx(ctx).Info("HeapObjects = ", bToMb(m.HeapObjects), " MB")
+	logger.Ctx(ctx).Info("HeapSys = ", bToMb(m.HeapSys), " MB")
+	logger.Ctx(ctx).Info("HeapIdle = ", bToMb(m.HeapIdle), " MB")
+	logger.Ctx(ctx).Info("HeapInuse = ", bToMb(m.HeapInuse), " MB")
+
+	// logger.Ctx(ctx).Info("Mallocs = ", bToMb(m.Mallocs), " MB")
+	// logger.Ctx(ctx).Info("Frees = ", bToMb(m.Frees), " MB")
+
+	// logger.Ctx(ctx).Info("StackInuse = ", bToMb(m.StackInuse), " MB")
+	// logger.Ctx(ctx).Info("StackSys = ", bToMb(m.StackSys), " MB")
+
+	// logger.Ctx(ctx).Info("Sys = ", bToMb(m.Sys), " MB")
+	logger.Ctx(ctx).Info("NumGC = ", m.NumGC)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}

src/go.mod

@@ -3,6 +3,8 @@ module github.com/alcionai/corso/src
 go 1.21
 
 replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230822191057-17d4deff94a3
+//replace github.com/DmitriyVTitov/size => /Users/pandeyab/temp/size
+
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1
@@ -46,15 +48,18 @@ require (
 )
 
 require (
+	github.com/DmitriyVTitov/size v1.5.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/aws/aws-sdk-go v1.47.9 // indirect
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/cronexpr v1.1.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -64,6 +69,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pkg/profile v1.7.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect

src/go.sum

@@ -49,6 +49,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM=
 github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
+github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI=
 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
@@ -121,6 +123,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -155,6 +159,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -207,6 +212,9 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk=
+github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
@@ -241,6 +249,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
@@ -353,6 +362,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
@@ -631,6 +642,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

src/internal/m365/collection/drive/collection.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/spatialcurrent/go-lazy/pkg/lazy"
 
 	"github.com/alcionai/corso/src/internal/common/idname"
@@ -52,7 +51,7 @@ type Collection struct {
 	// represents
 	folderPath path.Path
 	// M365 IDs of file items within this collection
-	driveItems map[string]models.DriveItemable
+	driveItems map[string]LiteDriveItemable
 
 	// Primary M365 ID of the drive this collection was created from
 	driveID   string
@@ -92,6 +91,10 @@ type Collection struct {
 	counter *count.Bus
 }
 
+func (c *Collection) GetDriveItemsMap() map[string]LiteDriveItemable {
+	return c.driveItems
+}
+
 func pathToLocation(p path.Path) (*path.Builder, error) {
 	if p == nil {
 		return nil, nil
@@ -172,7 +175,7 @@ func newColl(
 		protectedResource:         resource,
 		folderPath:                currPath,
 		prevPath:                  prevPath,
-		driveItems:                map[string]models.DriveItemable{},
+		driveItems:                map[string]LiteDriveItemable{},
 		driveID:                   driveID,
 		data:                      dataCh,
 		statusUpdater:             statusUpdater,
@@ -190,9 +193,13 @@ func newColl(
 // Adds an itemID to the collection.  This will make it eligible to be
 // populated. The return values denotes if the item was previously
 // present or is new one.
-func (oc *Collection) Add(item models.DriveItemable) bool {
-	_, found := oc.driveItems[ptr.Val(item.GetId())]
-	oc.driveItems[ptr.Val(item.GetId())] = item
+func (oc *Collection) Add(cdi LiteDriveItemable) bool {
+	// _, found := oc.driveItems[ptr.Val(item.GetId())]
+	// oc.driveItems[ptr.Val(item.GetId())] = item
+
+	//cdi := ToLiteDriveItemable(item)
+	_, found := oc.driveItems[ptr.Val(cdi.GetId())]
+	oc.driveItems[ptr.Val(cdi.GetId())] = cdi
 
 	// if !found, it's a new addition
 	return !found
@@ -261,13 +268,13 @@ func (oc Collection) DoNotMergeItems() bool {
 func (oc *Collection) getDriveItemContent(
 	ctx context.Context,
 	driveID string,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 	errs *fault.Bus,
 ) (io.ReadCloser, error) {
-	var (
-		itemID   = ptr.Val(item.GetId())
-		itemName = ptr.Val(item.GetName())
-	)
+	// var (
+	// 	itemID   = ptr.Val(item.GetId())
+	// 	itemName = ptr.Val(item.GetName())
+	// )
 
 	itemData, err := downloadContent(
 		ctx,
@@ -279,7 +286,7 @@ func (oc *Collection) getDriveItemContent(
 	if err != nil {
 		if clues.HasLabel(err, graph.LabelsMalware) || (item != nil && item.GetMalware() != nil) {
 			logger.CtxErr(ctx, err).With("skipped_reason", fault.SkipMalware).Info("item flagged as malware")
-			errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
+			//errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
 
 			return nil, clues.Wrap(err, "malware item").Label(graph.LabelsSkippable)
 		}
@@ -309,12 +316,12 @@ func (oc *Collection) getDriveItemContent(
 				CtxErr(ctx, err).
 				With("skipped_reason", fault.SkipOneNote).
 				Info("inaccessible one note file")
-			errs.AddSkip(ctx, fault.FileSkip(
-				fault.SkipOneNote,
-				driveID,
-				itemID,
-				itemName,
-				graph.ItemInfo(item)))
+			// errs.AddSkip(ctx, fault.FileSkip(
+			// 	fault.SkipOneNote,
+			// 	driveID,
+			// 	itemID,
+			// 	itemName,
+			// 	graph.ItemInfo(item)))
 
 			return nil, clues.Wrap(err, "inaccesible oneNote item").Label(graph.LabelsSkippable)
 		}
@@ -344,7 +351,7 @@ func downloadContent(
 	ctx context.Context,
 	iaag itemAndAPIGetter,
 	uc getItemPropertyer,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 	driveID string,
 	counter *count.Bus,
 ) (io.ReadCloser, error) {
@@ -379,7 +386,9 @@ func downloadContent(
 		return nil, clues.Wrap(err, "retrieving expired item")
 	}
 
-	content, err = downloadItem(ctx, iaag, di)
+	cdi := ToLiteDriveItemable(di)
+
+	content, err = downloadItem(ctx, iaag, cdi)
 	if err != nil {
 		return nil, clues.Wrap(err, "content download retry")
 	}
@@ -473,7 +482,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 
 		wg.Add(1)
 
-		go func(item models.DriveItemable) {
+		go func(item LiteDriveItemable) {
 			defer wg.Done()
 			defer func() { <-semaphoreCh }()
 
@@ -497,14 +506,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 
 type lazyItemGetter struct {
 	info                 *details.ItemInfo
-	item                 models.DriveItemable
+	item                 LiteDriveItemable
 	driveID              string
 	suffix               string
 	itemExtensionFactory []extensions.CreateItemExtensioner
 	contentGetter        func(
 		ctx context.Context,
 		driveID string,
-		item models.DriveItemable,
+		item LiteDriveItemable,
 		errs *fault.Bus) (io.ReadCloser, error)
 }
 
@@ -545,7 +554,7 @@ func (lig *lazyItemGetter) GetData(
 func (oc *Collection) streamDriveItem(
 	ctx context.Context,
 	parentPath *path.Builder,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 	stats *driveStats,
 	itemExtensionFactory []extensions.CreateItemExtensioner,
 	errs *fault.Bus,

src/internal/m365/collection/drive/collections.go

@@ -696,7 +696,7 @@ func (c *Collections) handleDelete(
 
 func (c *Collections) getCollectionPath(
 	driveID string,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 ) (path.Path, error) {
 	var (
 		pb     = odConsts.DriveFolderPrefixBuilder(driveID)
@@ -845,7 +845,7 @@ func (c *Collections) PopulateDriveCollections(
 
 func (c *Collections) processItem(
 	ctx context.Context,
-	item models.DriveItemable,
+	di models.DriveItemable,
 	driveID, driveName string,
 	oldPrevPaths, currPrevPaths, newPrevPaths map[string]string,
 	seenFolders map[string]string,
@@ -855,6 +855,8 @@ func (c *Collections) processItem(
 	counter *count.Bus,
 	skipper fault.AddSkipper,
 ) error {
+	item := ToLiteDriveItemable(di)
+
 	var (
 		itemID   = ptr.Val(item.GetId())
 		itemName = ptr.Val(item.GetName())
@@ -868,7 +870,7 @@ func (c *Collections) processItem(
 		"item_is_folder", isFolder)
 
 	if item.GetMalware() != nil {
-		addtl := graph.ItemInfo(item)
+		addtl := graph.ItemInfo(di)
 		skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
 
 		if isFolder {

src/internal/m365/collection/drive/custom_drive_item.go

@@ -0,0 +1,284 @@
+// Disable revive linter since any structs in this file will expose the same
+// funcs as the original structs in the msgraph-sdk-go package, which do not
+// follow some of the golint rules.
+//
+//nolint:revive
+package drive
+
+import (
+	"strings"
+	"time"
+
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/alcionai/corso/src/internal/common/str"
+)
+
+// Replica of models.DriveItemable
+type LiteDriveItemable interface {
+	GetId() *string
+	GetName() *string
+	GetSize() *int64
+	// TODO(pandeyabs): replace with any
+	GetFolder() interface{}
+	GetPackageEscaped() interface{}
+	GetShared() interface{}
+	GetMalware() interface{}
+	GetDeleted() interface{}
+	GetRoot() interface{}
+	GetFile() *fileItema
+	GetParentReference() parentReferenceable
+	SetParentReference(parentReferenceable)
+	GetCreatedBy() itemIdentitySetable
+	GetCreatedDateTime() *time.Time
+	GetLastModifiedDateTime() *time.Time
+	GetAdditionalData() map[string]interface{}
+}
+
+var _ LiteDriveItemable = &driveItema{}
+
+type driveItema struct {
+	id                   *string
+	name                 *string
+	size                 *int64
+	folder               interface{}
+	pkg                  interface{}
+	shared               interface{}
+	malware              interface{}
+	deleted              interface{}
+	root                 interface{}
+	file                 *fileItema
+	parentRef            parentReferenceable
+	createdBy            itemIdentitySetable
+	createdDateTime      *time.Time
+	lastModifiedDateTime *time.Time
+	additionalData       map[string]interface{}
+}
+
+// nolint
+func (c *driveItema) GetId() *string {
+	return c.id
+}
+
+func (c *driveItema) GetName() *string {
+	return c.name
+}
+
+func (c *driveItema) GetSize() *int64 {
+	return c.size
+}
+
+func (c *driveItema) GetFolder() interface{} {
+	return c.folder
+}
+
+func (c *driveItema) GetPackageEscaped() interface{} {
+	return c.pkg
+}
+
+func (c *driveItema) GetShared() interface{} {
+	return c.shared
+}
+
+func (c *driveItema) GetMalware() interface{} {
+	return c.malware
+}
+
+func (c *driveItema) GetDeleted() interface{} {
+	return c.deleted
+}
+
+func (c *driveItema) GetRoot() interface{} {
+	return c.root
+}
+
+func (c *driveItema) GetFile() *fileItema {
+	return c.file
+}
+
+func (c *driveItema) GetParentReference() parentReferenceable {
+	return c.parentRef
+}
+
+// TODO(pandeyabs): Should we only support GETs?
+func (c *driveItema) SetParentReference(parent parentReferenceable) {
+	c.parentRef = parent
+}
+
+func (c *driveItema) GetCreatedBy() itemIdentitySetable {
+	return c.createdBy
+}
+
+func (c *driveItema) GetCreatedDateTime() *time.Time {
+	return c.createdDateTime
+}
+
+func (c *driveItema) GetLastModifiedDateTime() *time.Time {
+	return c.lastModifiedDateTime
+}
+
+func (c *driveItema) GetAdditionalData() map[string]interface{} {
+	return c.additionalData
+}
+
+type (
+	parentReferenceable interface {
+		GetPath() *string
+		GetId() *string
+		GetName() *string
+		GetDriveId() *string
+	}
+	itemIdentitySetable interface {
+		GetUser() itemUserable
+	}
+	itemUserable interface {
+		GetAdditionalData() map[string]interface{}
+	}
+)
+
+// Concrete implementations
+
+type fileItema struct {
+	mimeType string
+}
+
+func (f *fileItema) GetMimeType() *string {
+	return &f.mimeType
+}
+
+var _ parentReferenceable = &parentRef{}
+
+type parentRef struct {
+	path    string
+	id      string
+	name    string
+	driveID string
+}
+
+func (pr *parentRef) GetPath() *string {
+	return &pr.path
+}
+
+func (pr *parentRef) GetId() *string {
+	return &pr.id
+}
+
+func (pr *parentRef) GetName() *string {
+	return &pr.name
+}
+
+func (pr *parentRef) GetDriveId() *string {
+	return &pr.driveID
+}
+
+var _ itemIdentitySetable = &itemIdentitySet{}
+
+type itemIdentitySet struct {
+	user itemUserable
+}
+
+func (iis *itemIdentitySet) GetUser() itemUserable {
+	return iis.user
+}
+
+var _ itemUserable = &itemUser{}
+
+type itemUser struct {
+	additionalData map[string]interface{}
+}
+
+func (iu *itemUser) GetAdditionalData() map[string]interface{} {
+	return iu.additionalData
+}
+
+func ToLiteDriveItemable(item models.DriveItemable) LiteDriveItemable {
+	cdi := &driveItema{}
+
+	id := strings.Clone(ptr.Val(item.GetId()))
+	name := strings.Clone(ptr.Val(item.GetName()))
+	size := ptr.Val(item.GetSize())
+	createdDateTime := ptr.Val(item.GetCreatedDateTime())
+	lastModifiedDateTime := ptr.Val(item.GetLastModifiedDateTime())
+
+	cdi.id = &id
+	cdi.name = &name
+	cdi.size = &size
+	cdi.createdDateTime = &createdDateTime
+	cdi.lastModifiedDateTime = &lastModifiedDateTime
+
+	if item.GetFolder() != nil {
+		cdi.folder = &struct{}{}
+	} else if item.GetFile() != nil {
+		cdi.file = &fileItema{
+			mimeType: strings.Clone(ptr.Val(item.GetFile().GetMimeType())),
+		}
+	} else if item.GetPackageEscaped() != nil {
+		cdi.pkg = &struct{}{}
+	}
+
+	if item.GetParentReference() != nil {
+		cdi.parentRef = &parentRef{
+			id:      strings.Clone(ptr.Val(item.GetParentReference().GetId())),
+			path:    strings.Clone(ptr.Val(item.GetParentReference().GetPath())),
+			name:    strings.Clone(ptr.Val(item.GetParentReference().GetName())),
+			driveID: strings.Clone(ptr.Val(item.GetParentReference().GetDriveId())),
+		}
+	}
+
+	if item.GetShared() != nil {
+		cdi.shared = &struct{}{}
+	}
+
+	if item.GetMalware() != nil {
+		cdi.malware = &struct{}{}
+	}
+
+	if item.GetDeleted() != nil {
+		cdi.deleted = &struct{}{}
+	}
+
+	if item.GetRoot() != nil {
+		cdi.root = &struct{}{}
+	}
+
+	if item.GetCreatedBy() != nil && item.GetCreatedBy().GetUser() != nil {
+		additionalData := item.GetCreatedBy().GetUser().GetAdditionalData()
+		ad := make(map[string]interface{})
+
+		var s string
+
+		ed, ok := additionalData["email"]
+		if ok {
+			s = strings.Clone(ptr.Val(ed.(*string)))
+			ad["email"] = &s
+		} else if ed, ok = additionalData["displayName"]; ok {
+			s = strings.Clone(ptr.Val(ed.(*string)))
+			ad["displayName"] = &s
+		}
+
+		cdi.createdBy = &itemIdentitySet{
+			user: &itemUser{
+				additionalData: ad,
+			},
+		}
+	}
+
+	// Hacky way to cache the download url. Thats all we use from additional data
+	// Otherwise, we'll hold a reference to the underlying store which will consume
+	// lot more memory.
+	if item.GetFile() != nil {
+		ad := make(map[string]interface{})
+
+		for _, key := range downloadURLKeys {
+			if v, err := str.AnyValueToString(key, item.GetAdditionalData()); err == nil {
+				ad[key] = strings.Clone(v)
+				break
+			}
+		}
+
+		cdi.additionalData = ad
+	}
+
+	return cdi
+}

src/internal/m365/collection/drive/group_handler.go

@@ -2,7 +2,6 @@ package drive
 
 import (
 	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/common/ptr"
@@ -105,7 +104,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) {
 func (h groupBackupHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/m365/collection/drive/handler_utils.go

@@ -3,12 +3,10 @@ package drive
 import (
 	"strings"
 
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
-
 	"github.com/alcionai/corso/src/internal/common/ptr"
 )
 
-func getItemCreator(item models.DriveItemable) string {
+func getItemCreator(item LiteDriveItemable) string {
 	if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil {
 		return ""
 	}
@@ -30,7 +28,7 @@ func getItemCreator(item models.DriveItemable) string {
 	return *ed.(*string)
 }
 
-func getItemDriveInfo(item models.DriveItemable) (string, string) {
+func getItemDriveInfo(item LiteDriveItemable) (string, string) {
 	if item.GetParentReference() == nil {
 		return "", ""
 	}

src/internal/m365/collection/drive/handlers.go

@@ -23,7 +23,7 @@ type ItemInfoAugmenter interface {
 	AugmentItemInfo(
 		dii details.ItemInfo,
 		resource idname.Provider,
-		item models.DriveItemable,
+		item LiteDriveItemable,
 		size int64,
 		parentPath *path.Builder,
 	) details.ItemInfo

src/internal/m365/collection/drive/item.go

@@ -34,7 +34,7 @@ var downloadURLKeys = []string{
 func downloadItem(
 	ctx context.Context,
 	ag api.Getter,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 ) (io.ReadCloser, error) {
 	if item == nil {
 		return nil, clues.New("nil item")
@@ -152,7 +152,7 @@ func downloadItemMeta(
 	ctx context.Context,
 	getter GetItemPermissioner,
 	driveID string,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 ) (io.ReadCloser, int, error) {
 	meta := metadata.Metadata{
 		FileName:    ptr.Val(item.GetName()),
@@ -205,12 +205,16 @@ func driveItemWriter(
 	return iw, ptr.Val(icu.GetUploadUrl()), nil
 }
 
-func setName(orig models.ItemReferenceable, driveName string) models.ItemReferenceable {
+func setName(orig parentReferenceable, driveName string) models.ItemReferenceable {
 	if orig == nil {
 		return nil
 	}
 
-	orig.SetName(&driveName)
+	mod := models.NewItemReference()
+	mod.SetDriveId(orig.GetDriveId())
+	mod.SetId(orig.GetId())
+	mod.SetPath(orig.GetPath())
+	mod.SetName(&driveName)
 
-	return orig
+	return mod
 }

src/internal/m365/collection/drive/restore.go

@@ -863,7 +863,7 @@ func restoreFile(
 	dii := ir.AugmentItemInfo(
 		details.ItemInfo{},
 		rcc.ProtectedResource,
-		newItem,
+		nil,
 		written,
 		nil)
 

src/internal/m365/collection/drive/site_handler.go

@@ -33,7 +33,7 @@ func (h baseSiteHandler) NewDrivePager(
 func (h baseSiteHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/m365/collection/drive/user_drive_handler.go

@@ -42,7 +42,7 @@ func (h baseUserDriveHandler) NewDrivePager(
 func (h baseUserDriveHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item LiteDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/operations/backup.go

@@ -439,6 +439,9 @@ func (op *BackupOperation) do(
 		lastBackupVersion = mans.MinBackupVersion()
 	}
 
+	// iterations := 1
+
+	// for i := 0; i < iterations; i++ {
 	// TODO(ashmrtn): This should probably just return a collection that deletes
 	// the entire subtree instead of returning an additional bool. That way base
 	// selection is controlled completely by flags and merging is controlled
@@ -457,6 +460,46 @@ func (op *BackupOperation) do(
 		return nil, clues.Wrap(err, "producing backup data collections")
 	}
 
+	// Sleep for 4 mins to let the memory usage settle down so that we have a better
+	// picture. Also allows pprof to run twice during this time.
+	// Do some meaningless work after to make sure the collections dont get garbage collected
+	time.Sleep(4 * time.Minute)
+
+	// sum := 0
+	// numItems := 0
+	// mapSum := 0
+
+	// for _, c := range cs {
+	// 	v, ok := c.(*drive.Collection)
+	// 	if !ok {
+	// 		continue
+	// 	}
+
+	// 	m := v.GetDriveItemsMap()
+	// 	for key := range m {
+	// 		logger.Ctx(ctx).Debug(key)
+	// 	}
+
+	// 	// Get sizeof recursively using reflect
+	// 	// m := v.GetDriveItemsMap()
+	// 	// for _, val := range m {
+	// 	// 	s := size.Of(val)
+	// 	// 	sum += s
+	// 	// 	numItems++
+	// 	// }
+
+	// 	// ms := size.Of(m)
+	// 	// mapSum += ms
+
+	// 	// logger.Ctx(ctx).Debugf("coll drive map size %d, num drive items %d\n", ms, len(m))
+	// }
+
+	// print total sum
+	// logger.Ctx(ctx).Debugf("itemSum %d, map sum %d, total items %d, mem used per item %f mem per item in map %f \n", sum, mapSum, numItems, float64(sum)/float64(numItems), float64(mapSum)/float64(numItems))
+	//}
+
+	// return nil, clues.New("failed")
+
 	ctx = clues.Add(
 		ctx,
 		"can_use_previous_backup", canUsePreviousBackup,