1970 11 tracker fails redo (#2624)

#### Type of change

- [x] 🐛 Bugfix

#### Issue(s)

* #1970

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E

src/internal/connector/exchange/api/contacts.go

@@ -126,24 +126,33 @@ func (c Contacts) EnumerateContainers(
 			With("options_fields", fields)
 	}
 
+	et := errs.Tracker()
 	builder := service.Client().
 		UsersById(userID).
 		ContactFoldersById(baseDirID).
 		ChildFolders()
 
 	for {
+		if et.Err() != nil {
+			break
+		}
+
 		resp, err := builder.Get(ctx, ofcf)
 		if err != nil {
 			return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
 		}
 
 		for _, fold := range resp.GetValue() {
-			if errs.Err() != nil {
-				return errs.Err()
+			if et.Err() != nil {
+				break
 			}
 
 			if err := checkIDAndName(fold); err != nil {
-				errs.Add(clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...))
+				et.Add(clues.Stack(err).
+					WithClues(ctx).
+					With(graph.ErrData(err)...).
+					Label(fault.LabelForceNoBackupCreation))
+
 				continue
 			}
 
@@ -154,7 +163,11 @@ func (c Contacts) EnumerateContainers(
 
 			temp := graph.NewCacheFolder(fold, nil, nil)
 			if err := fn(temp); err != nil {
-				errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...))
+				et.Add(clues.Stack(err).
+					WithClues(fctx).
+					With(graph.ErrData(err)...).
+					Label(fault.LabelForceNoBackupCreation))
+
 				continue
 			}
 		}
@@ -167,7 +180,7 @@ func (c Contacts) EnumerateContainers(
 		builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(link, service.Adapter())
 	}
 
-	return errs.Err()
+	return et.Err()
 }
 
 // ---------------------------------------------------------------------------

src/internal/connector/exchange/api/events.go

@@ -153,18 +153,31 @@ func (c Events) EnumerateContainers(
 		return clues.Wrap(err, "setting calendar options").WithClues(ctx).With(graph.ErrData(err)...)
 	}
 
+	et := errs.Tracker()
 	builder := service.Client().UsersById(userID).Calendars()
 
 	for {
+		if et.Err() != nil {
+			break
+		}
+
 		resp, err := builder.Get(ctx, ofc)
 		if err != nil {
 			return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
 		}
 
 		for _, cal := range resp.GetValue() {
+			if et.Err() != nil {
+				break
+			}
+
 			cd := CalendarDisplayable{Calendarable: cal}
 			if err := checkIDAndName(cd); err != nil {
-				errs.Add(clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...))
+				et.Add(clues.Stack(err).
+					WithClues(ctx).
+					With(graph.ErrData(err)...).
+					Label(fault.LabelForceNoBackupCreation))
+
 				continue
 			}
 
@@ -178,7 +191,11 @@ func (c Events) EnumerateContainers(
 				path.Builder{}.Append(ptr.Val(cd.GetId())),          // storage path
 				path.Builder{}.Append(ptr.Val(cd.GetDisplayName()))) // display location
 			if err := fn(temp); err != nil {
-				errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...))
+				et.Add(clues.Stack(err).
+					WithClues(fctx).
+					With(graph.ErrData(err)...).
+					Label(fault.LabelForceNoBackupCreation))
+
 				continue
 			}
 		}
@@ -191,7 +208,7 @@ func (c Events) EnumerateContainers(
 		builder = users.NewItemCalendarsRequestBuilder(link, service.Adapter())
 	}
 
-	return errs.Err()
+	return et.Err()
 }
 
 // ---------------------------------------------------------------------------

src/internal/connector/exchange/api/mail.go

@@ -171,18 +171,27 @@ func (c Mail) EnumerateContainers(
 		return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
 	}
 
+	et := errs.Tracker()
 	builder := service.Client().
 		UsersById(userID).
 		MailFolders().
 		Delta()
 
 	for {
+		if et.Err() != nil {
+			break
+		}
+
 		resp, err := builder.Get(ctx, nil)
 		if err != nil {
 			return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
 		}
 
 		for _, v := range resp.GetValue() {
+			if et.Err() != nil {
+				break
+			}
+
 			fctx := clues.Add(
 				ctx,
 				"container_id", ptr.Val(v.GetId()),
@@ -190,7 +199,11 @@ func (c Mail) EnumerateContainers(
 
 			temp := graph.NewCacheFolder(v, nil, nil)
 			if err := fn(temp); err != nil {
-				errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...))
+				et.Add(clues.Stack(err).
+					WithClues(fctx).
+					With(graph.ErrData(err)...).
+					Label(fault.LabelForceNoBackupCreation))
+
 				continue
 			}
 		}
@@ -203,7 +216,7 @@ func (c Mail) EnumerateContainers(
 		builder = users.NewItemMailFoldersDeltaRequestBuilder(link, service.Adapter())
 	}
 
-	return errs.Err()
+	return et.Err()
 }
 
 // ---------------------------------------------------------------------------

src/internal/connector/exchange/exchange_data_collection.go

@@ -253,7 +253,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
 					atomic.AddInt64(&success, 1)
 					log.With("err", err).Infow("item not found", clues.InErr(err).Slice()...)
 				} else {
-					errs.Add(clues.Wrap(err, "fetching item"))
+					errs.Add(clues.Wrap(err, "fetching item").Label(fault.LabelForceNoBackupCreation))
 				}
 
 				return
@@ -261,7 +261,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
 
 			data, err := col.items.Serialize(ctx, item, user, id)
 			if err != nil {
-				errs.Add(clues.Wrap(err, "serializing item"))
+				errs.Add(clues.Wrap(err, "serializing item").Label(fault.LabelForceNoBackupCreation))
 				return
 			}
 

src/internal/connector/exchange/service_iterators.go

@@ -102,7 +102,7 @@ func filterContainersAndFillCollections(
 		added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ctx, qp.ResourceOwner, cID, prevDelta)
 		if err != nil {
 			if !graph.IsErrDeletedInFlight(err) {
-				et.Add(err)
+				et.Add(clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
 				continue
 			}
 

src/internal/connector/graph/metadata_collection.go

@@ -134,7 +134,7 @@ func (md MetadataCollection) DoNotMergeItems() bool {
 
 func (md MetadataCollection) Items(
 	ctx context.Context,
-	errs *fault.Errors,
+	_ *fault.Errors, // not used, just here for interface compliance
 ) <-chan data.Stream {
 	res := make(chan data.Stream)
 

src/internal/connector/onedrive/collection.go

@@ -273,6 +273,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
 
 	errUpdater := func(id string, err error) {
 		m.Lock()
+		// TODO: Label(fault.LabelForceNoBackupCreation)
 		errs = support.WrapAndAppend(id, err, errs)
 		m.Unlock()
 	}
@@ -451,6 +452,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
 func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64, errs error) {
 	close(oc.data)
 
+	// TODO: add Label(fault.LabelForceNoBackupCreation) to errs
 	status := support.CreateStatus(ctx, support.Backup,
 		1, // num folders (always 1)
 		support.CollectionMetrics{

src/internal/connector/sharepoint/collection.go

@@ -243,7 +243,7 @@ func (sc *Collection) retrieveLists(
 
 		byteArray, err := serializeContent(wtr, lst)
 		if err != nil {
-			et.Add(clues.Wrap(err, "serializing list").WithClues(ctx))
+			et.Add(clues.Wrap(err, "serializing list").WithClues(ctx).Label(fault.LabelForceNoBackupCreation))
 			continue
 		}
 
@@ -311,7 +311,7 @@ func (sc *Collection) retrievePages(
 
 		byteArray, err := serializeContent(wtr, pg)
 		if err != nil {
-			et.Add(clues.Wrap(err, "serializing page").WithClues(ctx))
+			et.Add(clues.Wrap(err, "serializing page").WithClues(ctx).Label(fault.LabelForceNoBackupCreation))
 			continue
 		}
 

src/internal/kopia/data_collection.go

@@ -26,7 +26,7 @@ type kopiaDataCollection struct {
 
 func (kdc *kopiaDataCollection) Items(
 	ctx context.Context,
-	errs *fault.Errors,
+	_ *fault.Errors, // unused, just matching the interface
 ) <-chan data.Stream {
 	res := make(chan data.Stream)
 

src/internal/kopia/upload.go

@@ -264,7 +264,8 @@ func (cp *corsoProgress) Error(relpath string, err error, isIgnored bool) {
 	defer cp.UploadProgress.Error(relpath, err, isIgnored)
 
 	cp.errs.Add(clues.Wrap(err, "kopia reported error").
-		With("is_ignored", isIgnored, "relative_path", relpath))
+		With("is_ignored", isIgnored, "relative_path", relpath).
+		Label(fault.LabelForceNoBackupCreation))
 }
 
 func (cp *corsoProgress) put(k string, v *itemDetails) {

src/internal/kopia/wrapper.go

@@ -410,13 +410,16 @@ func (w Wrapper) RestoreMultipleItems(
 
 		ds, err := getItemStream(ctx, itemPath, snapshotRoot, bcounter)
 		if err != nil {
-			et.Add(err)
+			et.Add(clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
 			continue
 		}
 
 		parentPath, err := itemPath.Dir()
 		if err != nil {
-			et.Add(clues.Wrap(err, "making directory collection").WithClues(ctx))
+			et.Add(clues.Wrap(err, "making directory collection").
+				WithClues(ctx).
+				Label(fault.LabelForceNoBackupCreation))
+
 			continue
 		}
 

src/internal/operations/backup.go

@@ -190,6 +190,20 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		return op.Errors.Err()
 	}
 
+	// force exit without backup in certain cases.
+	// see: https://github.com/alcionai/corso/pull/2510#discussion_r1113532530
+	for _, e := range op.Errors.Errs() {
+		if clues.HasLabel(e, fault.LabelForceNoBackupCreation) {
+			logger.Ctx(ctx).
+				With("error", e).
+				With(clues.InErr(err).Slice()...).
+				Infow("completed backup; conditional error forcing exit without model persistence",
+					"results", op.Results)
+
+			return op.Errors.Fail(errors.Wrap(e, "forced backup")).Err()
+		}
+	}
+
 	err = op.createBackupModels(
 		ctx,
 		detailsStore,

src/pkg/fault/fault.go

@@ -169,3 +169,7 @@ func (e *tracker) Add(err error) {
 func (e *tracker) Err() error {
 	return e.current
 }
+
+// temporary hack identifier
+// see: https://github.com/alcionai/corso/pull/2510#discussion_r1113532530
+const LabelForceNoBackupCreation = "label_forces_no_backup_creations"