Only delta queries

src/corso.go

@@ -15,7 +15,7 @@ import (
 )
 
 var profileTicker = time.NewTicker(300 * time.Second)
-var printTicker = time.NewTicker(10 * time.Second)
+var printTicker = time.NewTicker(1 * time.Second)
 var profileCounter = 0
 
 func main() {
@@ -67,6 +67,7 @@ func PrintMemUsage() {
 	logger.Ctx(ctx).Info("HeapObjects = ", bToMb(m.HeapObjects), " MB")
 	logger.Ctx(ctx).Info("HeapSys = ", bToMb(m.HeapSys), " MB")
 	logger.Ctx(ctx).Info("HeapIdle = ", bToMb(m.HeapIdle), " MB")
+	logger.Ctx(ctx).Info("HeapInuse = ", bToMb(m.HeapInuse), " MB")
 
 	logger.Ctx(ctx).Info("Mallocs = ", bToMb(m.Mallocs), " MB")
 	logger.Ctx(ctx).Info("Frees = ", bToMb(m.Frees), " MB")

src/internal/operations/backup.go

@@ -369,27 +369,33 @@ func (op *BackupOperation) do(
 		lastBackupVersion = mans.MinBackupVersion()
 	}
 
-	// TODO(ashmrtn): This should probably just return a collection that deletes
+	// Run 3 times and exit
-	// the entire subtree instead of returning an additional bool. That way base
+	cs := []data.BackupCollection{}
-	// selection is controlled completely by flags and merging is controlled
+	canUsePreviousBackup := false
-	// completely by collections.
-	cs, ssmb, canUsePreviousBackup, err := produceBackupDataCollections(
-		ctx,
-		op.bp,
-		op.ResourceOwner,
-		op.Selectors,
-		mdColls,
-		lastBackupVersion,
-		op.Options,
-		op.Errors)
-	if err != nil {
-		return nil, clues.Wrap(err, "producing backup data collections")
-	}
 
-	ctx = clues.Add(
+	for i := 0; i < 3; i++ {
-		ctx,
+		cs, _, canUsePreviousBackup, err := produceBackupDataCollections(
-		"can_use_previous_backup", canUsePreviousBackup,
+			ctx,
-		"collection_count", len(cs))
+			op.bp,
+			op.ResourceOwner,
+			op.Selectors,
+			mdColls,
+			lastBackupVersion,
+			op.Options,
+			op.Errors)
+		if err != nil {
+			return nil, clues.Wrap(err, "producing backup data collections")
+		}
+
+		ctx = clues.Add(
+			ctx,
+			"can_use_previous_backup", canUsePreviousBackup,
+			"collection_count", len(cs))
+
+		if i == 2 {
+			return nil, clues.New("unable to produce backup collections").WithClues(ctx)
+		}
+	}
 
 	writeStats, deets, toMerge, err := consumeBackupCollections(
 		ctx,
@@ -398,7 +404,7 @@ func (op *BackupOperation) do(
 		reasons,
 		mans,
 		cs,
-		ssmb,
+		nil,
 		backupID,
 		op.incremental && canUseMetadata && canUsePreviousBackup,
 		op.Errors)