Add both dq and udq

src/corso.go

@@ -1,9 +1,84 @@
 package main
 
 import (
+	"context"
+	"log"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"strconv"
+	"time"
+
 	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/pkg/profile"
 )
 
+var profileTicker = time.NewTicker(300 * time.Second)
+var printTicker = time.NewTicker(1 * time.Second)
+var profileCounter = 0
+
 func main() {
+	defer profile.Start(profile.MemProfile).Stop()
+
+	go func() {
+		for {
+			select {
+			case <-profileTicker.C:
+				filename := "mem." + strconv.Itoa(profileCounter) + ".pprof"
+
+				f, _ := os.Create(filename)
+				if err := pprof.WriteHeapProfile(f); err != nil {
+					log.Fatal("could not write memory profile: ", err)
+				}
+
+				f.Close()
+
+				profileCounter++
+
+			}
+		}
+	}()
+
+	go func() {
+		for {
+			select {
+			case <-printTicker.C:
+				PrintMemUsage()
+			}
+		}
+	}()
+
 	cli.Handle()
 }
+
+// PrintMemUsage outputs the current, total and OS memory being used. As well as the number
+// of garage collection cycles completed.
+func PrintMemUsage() {
+	ctx := context.Background()
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	logger.Ctx(ctx).Info("Alloc = ", bToMb(m.Alloc), " MB")
+	logger.Ctx(ctx).Info("TotalAlloc = ", bToMb(m.TotalAlloc), " MB")
+	logger.Ctx(ctx).Info("HeapAlloc = ", bToMb(m.HeapAlloc), " MB") // same as Alloc
+
+	logger.Ctx(ctx).Info("HeapReleased = ", bToMb(m.HeapReleased), " MB")
+	logger.Ctx(ctx).Info("HeapObjects = ", bToMb(m.HeapObjects), " MB")
+	logger.Ctx(ctx).Info("HeapSys = ", bToMb(m.HeapSys), " MB")
+	logger.Ctx(ctx).Info("HeapIdle = ", bToMb(m.HeapIdle), " MB")
+	logger.Ctx(ctx).Info("HeapInuse = ", bToMb(m.HeapInuse), " MB")
+
+	logger.Ctx(ctx).Info("Mallocs = ", bToMb(m.Mallocs), " MB")
+	logger.Ctx(ctx).Info("Frees = ", bToMb(m.Frees), " MB")
+
+	logger.Ctx(ctx).Info("StackInuse = ", bToMb(m.StackInuse), " MB")
+	logger.Ctx(ctx).Info("StackSys = ", bToMb(m.StackSys), " MB")
+
+	logger.Ctx(ctx).Info("Sys = ", bToMb(m.Sys), " MB")
+	logger.Ctx(ctx).Info("NumGC = ", m.NumGC)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}

src/go.mod

@@ -42,9 +42,11 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
+	github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -99,6 +101,7 @@ require (
 	github.com/natefinch/atomic v1.0.1 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/profile v1.7.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect

src/go.sum

@@ -112,6 +112,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
@@ -190,6 +192,9 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
@@ -219,6 +224,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -319,6 +325,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -588,6 +596,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

src/internal/m365/collection/drive/collections.go

@@ -286,17 +286,25 @@ func (c *Collections) Get(
 			"num_paths_entries", len(oldPaths),
 			"num_deltas_entries", numOldDelta)
 
-		delta, paths, excluded, err := collectItems(
-			ictx,
-			c.handler.NewItemPager(driveID, "", api.DriveItemSelectDefault()),
-			driveID,
-			driveName,
-			c.UpdateCollections,
-			oldPaths,
-			prevDelta,
-			errs)
-		if err != nil {
-			return nil, false, err
+		var delta DeltaUpdate
+		var paths map[string]string
+		var excluded map[string]struct{}
+
+		for i := 0; i < 3; i++ {
+			logger.Ctx(ctx).Info("Delta query - original begin")
+			delta, paths, excluded, err = collectItems(
+				ictx,
+				c.handler.NewItemPager(driveID, "", api.DriveItemSelectDefault()),
+				driveID,
+				driveName,
+				c.UpdateCollections,
+				oldPaths,
+				prevDelta,
+				errs)
+			if err != nil {
+				return nil, false, err
+			}
+			logger.Ctx(ctx).Info("Delta query - original end")
 		}
 
 		// Used for logging below.
@@ -342,6 +350,29 @@ func (c *Collections) Get(
 			}
 		}
 
+		for i := 0; i < 3; i++ {
+			logger.Ctx(ctx).Info("Delta query - URL cache begin")
+			err = c.addURLCacheToDriveCollections(
+				ictx,
+				driveID,
+				prevDelta,
+				errs)
+			if err != nil {
+				return nil, false, err
+			}
+
+			var uc *urlCache
+			for _, driveColls := range c.CollectionMap {
+				for _, coll := range driveColls {
+					uc = coll.urlCache.(*urlCache)
+				}
+			}
+
+			uc.RefreshCache(ctx)
+
+			logger.Ctx(ctx).Info("Delta query - URL cache end")
+		}
+
 		// For both cases we don't need to do set difference on folder map if the
 		// delta token was valid because we should see all the changes.
 		if !delta.Reset {
@@ -471,7 +502,7 @@ func (c *Collections) addURLCacheToDriveCollections(
 		driveID,
 		prevDelta,
 		urlCacheRefreshInterval,
-		c.handler.NewItemPager(driveID, "", api.DriveItemSelectDefault()),
+		c.handler.NewItemPager(driveID, "", api.DriveItemSelectURLCache()),
 		errs)
 	if err != nil {
 		return err

src/internal/m365/collection/drive/url_cache.go

@@ -112,7 +112,7 @@ func (uc *urlCache) getItemProperties(
 	ctx = clues.Add(ctx, "drive_id", uc.driveID)
 
 	if uc.needsRefresh() {
-		err := uc.refreshCache(ctx)
+		err := uc.RefreshCache(ctx)
 		if err != nil {
 			return itemProps{}, err
 		}
@@ -137,7 +137,7 @@ func (uc *urlCache) needsRefresh() bool {
 }
 
 // refreshCache refreshes the URL cache by performing a delta query.
-func (uc *urlCache) refreshCache(
+func (uc *urlCache) RefreshCache(
 	ctx context.Context,
 ) error {
 	// Acquire mutex to prevent multiple threads from refreshing the

src/internal/m365/collection/drive/url_cache_test.go

@@ -87,6 +87,7 @@ func (suite *URLCacheIntegrationSuite) TestURLCacheBasic() {
 		newItem(newFolderName, true),
 		control.Copy)
 	require.NoError(t, err, clues.ToCore(err))
+
 	require.NotNil(t, newFolder.GetId())
 
 	nfid := ptr.Val(newFolder.GetId())
@@ -131,10 +132,7 @@ func (suite *URLCacheIntegrationSuite) TestURLCacheBasic() {
 			nfid,
 			newItem(newItemName, false),
 			control.Copy)
-		if err != nil {
-			// Something bad happened, skip this item
-			continue
-		}
+		require.NoError(t, err, clues.ToCore(err))
 
 		items = append(items, item)
 	}
@@ -176,13 +174,18 @@ func (suite *URLCacheIntegrationSuite) TestURLCacheBasic() {
 				nil,
 				nil)
 			require.NoError(t, err, clues.ToCore(err))
+
+			require.NotNil(t, resp)
+			require.NotNil(t, resp.Body)
 			require.Equal(t, http.StatusOK, resp.StatusCode)
+
+			resp.Body.Close()
 		}(i)
 	}
 	wg.Wait()
 
-	// Validate that <= 1 delta queries were made by url cache
-	require.LessOrEqual(t, uc.deltaQueryCount, 1)
+	// Validate that exactly 1 delta query was made by url cache
+	require.Equal(t, 1, uc.deltaQueryCount)
 }
 
 type URLCacheUnitSuite struct {

src/pkg/services/m365/api/config.go

@@ -112,3 +112,14 @@ func DriveItemSelectDefault() []string {
 		"malware",
 		"shared")
 }
+
+// URL cache only needs a subset of default select fields.
+// This improves memory usage for delta query operations done by URL cache.
+func DriveItemSelectURLCache() []string {
+	return idAnd(
+		"content.downloadUrl",
+		"deleted",
+		"file",
+		"folder",
+	)
+}