From 1459e1406cd825c839e971bee6ca413ad5dc3b15 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 24 Feb 2023 08:57:50 -0700 Subject: [PATCH] 1970 11 tracker fails redo (#2624) #### Type of change - [x] :bug: Bugfix #### Issue(s) * #1970 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../connector/exchange/api/contacts.go | 23 +++++++++++++++---- src/internal/connector/exchange/api/events.go | 23 ++++++++++++++++--- src/internal/connector/exchange/api/mail.go | 17 ++++++++++++-- .../exchange/exchange_data_collection.go | 4 ++-- .../connector/exchange/service_iterators.go | 2 +- .../connector/graph/metadata_collection.go | 2 +- src/internal/connector/onedrive/collection.go | 2 ++ .../connector/sharepoint/collection.go | 4 ++-- src/internal/kopia/data_collection.go | 2 +- src/internal/kopia/upload.go | 3 ++- src/internal/kopia/wrapper.go | 7 ++++-- src/internal/operations/backup.go | 14 +++++++++++ src/pkg/fault/fault.go | 4 ++++ 13 files changed, 87 insertions(+), 20 deletions(-) diff --git a/src/internal/connector/exchange/api/contacts.go b/src/internal/connector/exchange/api/contacts.go index ed766b800..0051afe4b 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/internal/connector/exchange/api/contacts.go @@ -126,24 +126,33 @@ func (c Contacts) EnumerateContainers( With("options_fields", fields) } + et := errs.Tracker() builder := service.Client(). UsersById(userID). ContactFoldersById(baseDirID). ChildFolders() for { + if et.Err() != nil { + break + } + resp, err := builder.Get(ctx, ofcf) if err != nil { return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...) } for _, fold := range resp.GetValue() { - if errs.Err() != nil { - return errs.Err() + if et.Err() != nil { + break } if err := checkIDAndName(fold); err != nil { - errs.Add(clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)) + et.Add(clues.Stack(err). + WithClues(ctx). + With(graph.ErrData(err)...). + Label(fault.LabelForceNoBackupCreation)) + continue } @@ -154,7 +163,11 @@ func (c Contacts) EnumerateContainers( temp := graph.NewCacheFolder(fold, nil, nil) if err := fn(temp); err != nil { - errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...)) + et.Add(clues.Stack(err). + WithClues(fctx). + With(graph.ErrData(err)...). + Label(fault.LabelForceNoBackupCreation)) + continue } } @@ -167,7 +180,7 @@ func (c Contacts) EnumerateContainers( builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(link, service.Adapter()) } - return errs.Err() + return et.Err() } // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index 99fdf4d5a..d3be3af46 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -153,18 +153,31 @@ func (c Events) EnumerateContainers( return clues.Wrap(err, "setting calendar options").WithClues(ctx).With(graph.ErrData(err)...) } + et := errs.Tracker() builder := service.Client().UsersById(userID).Calendars() for { + if et.Err() != nil { + break + } + resp, err := builder.Get(ctx, ofc) if err != nil { return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...) } for _, cal := range resp.GetValue() { + if et.Err() != nil { + break + } + cd := CalendarDisplayable{Calendarable: cal} if err := checkIDAndName(cd); err != nil { - errs.Add(clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)) + et.Add(clues.Stack(err). + WithClues(ctx). + With(graph.ErrData(err)...). + Label(fault.LabelForceNoBackupCreation)) + continue } @@ -178,7 +191,11 @@ func (c Events) EnumerateContainers( path.Builder{}.Append(ptr.Val(cd.GetId())), // storage path path.Builder{}.Append(ptr.Val(cd.GetDisplayName()))) // display location if err := fn(temp); err != nil { - errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...)) + et.Add(clues.Stack(err). + WithClues(fctx). + With(graph.ErrData(err)...). + Label(fault.LabelForceNoBackupCreation)) + continue } } @@ -191,7 +208,7 @@ func (c Events) EnumerateContainers( builder = users.NewItemCalendarsRequestBuilder(link, service.Adapter()) } - return errs.Err() + return et.Err() } // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index 37c8b0a91..a29aa11fb 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -171,18 +171,27 @@ func (c Mail) EnumerateContainers( return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...) } + et := errs.Tracker() builder := service.Client(). UsersById(userID). MailFolders(). Delta() for { + if et.Err() != nil { + break + } + resp, err := builder.Get(ctx, nil) if err != nil { return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...) } for _, v := range resp.GetValue() { + if et.Err() != nil { + break + } + fctx := clues.Add( ctx, "container_id", ptr.Val(v.GetId()), @@ -190,7 +199,11 @@ func (c Mail) EnumerateContainers( temp := graph.NewCacheFolder(v, nil, nil) if err := fn(temp); err != nil { - errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...)) + et.Add(clues.Stack(err). + WithClues(fctx). + With(graph.ErrData(err)...). + Label(fault.LabelForceNoBackupCreation)) + continue } } @@ -203,7 +216,7 @@ func (c Mail) EnumerateContainers( builder = users.NewItemMailFoldersDeltaRequestBuilder(link, service.Adapter()) } - return errs.Err() + return et.Err() } // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 4caec431a..a1082f7c1 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -253,7 +253,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) { atomic.AddInt64(&success, 1) log.With("err", err).Infow("item not found", clues.InErr(err).Slice()...) } else { - errs.Add(clues.Wrap(err, "fetching item")) + errs.Add(clues.Wrap(err, "fetching item").Label(fault.LabelForceNoBackupCreation)) } return @@ -261,7 +261,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) { data, err := col.items.Serialize(ctx, item, user, id) if err != nil { - errs.Add(clues.Wrap(err, "serializing item")) + errs.Add(clues.Wrap(err, "serializing item").Label(fault.LabelForceNoBackupCreation)) return } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 02f9d92b0..bc0398fbc 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -102,7 +102,7 @@ func filterContainersAndFillCollections( added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ctx, qp.ResourceOwner, cID, prevDelta) if err != nil { if !graph.IsErrDeletedInFlight(err) { - et.Add(err) + et.Add(clues.Stack(err).Label(fault.LabelForceNoBackupCreation)) continue } diff --git a/src/internal/connector/graph/metadata_collection.go b/src/internal/connector/graph/metadata_collection.go index c705eb4ce..cbb72250a 100644 --- a/src/internal/connector/graph/metadata_collection.go +++ b/src/internal/connector/graph/metadata_collection.go @@ -134,7 +134,7 @@ func (md MetadataCollection) DoNotMergeItems() bool { func (md MetadataCollection) Items( ctx context.Context, - errs *fault.Errors, + _ *fault.Errors, // not used, just here for interface compliance ) <-chan data.Stream { res := make(chan data.Stream) diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index 814e14965..f976a9a5d 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -273,6 +273,7 @@ func (oc *Collection) populateItems(ctx context.Context) { errUpdater := func(id string, err error) { m.Lock() + // TODO: Label(fault.LabelForceNoBackupCreation) errs = support.WrapAndAppend(id, err, errs) m.Unlock() } @@ -451,6 +452,7 @@ func (oc *Collection) populateItems(ctx context.Context) { func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64, errs error) { close(oc.data) + // TODO: add Label(fault.LabelForceNoBackupCreation) to errs status := support.CreateStatus(ctx, support.Backup, 1, // num folders (always 1) support.CollectionMetrics{ diff --git a/src/internal/connector/sharepoint/collection.go b/src/internal/connector/sharepoint/collection.go index 5e180c03d..8a2b11bab 100644 --- a/src/internal/connector/sharepoint/collection.go +++ b/src/internal/connector/sharepoint/collection.go @@ -243,7 +243,7 @@ func (sc *Collection) retrieveLists( byteArray, err := serializeContent(wtr, lst) if err != nil { - et.Add(clues.Wrap(err, "serializing list").WithClues(ctx)) + et.Add(clues.Wrap(err, "serializing list").WithClues(ctx).Label(fault.LabelForceNoBackupCreation)) continue } @@ -311,7 +311,7 @@ func (sc *Collection) retrievePages( byteArray, err := serializeContent(wtr, pg) if err != nil { - et.Add(clues.Wrap(err, "serializing page").WithClues(ctx)) + et.Add(clues.Wrap(err, "serializing page").WithClues(ctx).Label(fault.LabelForceNoBackupCreation)) continue } diff --git a/src/internal/kopia/data_collection.go b/src/internal/kopia/data_collection.go index f4c7a8313..d01eb10e5 100644 --- a/src/internal/kopia/data_collection.go +++ b/src/internal/kopia/data_collection.go @@ -26,7 +26,7 @@ type kopiaDataCollection struct { func (kdc *kopiaDataCollection) Items( ctx context.Context, - errs *fault.Errors, + _ *fault.Errors, // unused, just matching the interface ) <-chan data.Stream { res := make(chan data.Stream) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index c6d77b126..93ed6c503 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -264,7 +264,8 @@ func (cp *corsoProgress) Error(relpath string, err error, isIgnored bool) { defer cp.UploadProgress.Error(relpath, err, isIgnored) cp.errs.Add(clues.Wrap(err, "kopia reported error"). - With("is_ignored", isIgnored, "relative_path", relpath)) + With("is_ignored", isIgnored, "relative_path", relpath). + Label(fault.LabelForceNoBackupCreation)) } func (cp *corsoProgress) put(k string, v *itemDetails) { diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 732872c65..40871cd42 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -410,13 +410,16 @@ func (w Wrapper) RestoreMultipleItems( ds, err := getItemStream(ctx, itemPath, snapshotRoot, bcounter) if err != nil { - et.Add(err) + et.Add(clues.Stack(err).Label(fault.LabelForceNoBackupCreation)) continue } parentPath, err := itemPath.Dir() if err != nil { - et.Add(clues.Wrap(err, "making directory collection").WithClues(ctx)) + et.Add(clues.Wrap(err, "making directory collection"). + WithClues(ctx). + Label(fault.LabelForceNoBackupCreation)) + continue } diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index e0fe73870..b990c3d4c 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -190,6 +190,20 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { return op.Errors.Err() } + // force exit without backup in certain cases. + // see: https://github.com/alcionai/corso/pull/2510#discussion_r1113532530 + for _, e := range op.Errors.Errs() { + if clues.HasLabel(e, fault.LabelForceNoBackupCreation) { + logger.Ctx(ctx). + With("error", e). + With(clues.InErr(err).Slice()...). + Infow("completed backup; conditional error forcing exit without model persistence", + "results", op.Results) + + return op.Errors.Fail(errors.Wrap(e, "forced backup")).Err() + } + } + err = op.createBackupModels( ctx, detailsStore, diff --git a/src/pkg/fault/fault.go b/src/pkg/fault/fault.go index b67cb530a..b13f6916d 100644 --- a/src/pkg/fault/fault.go +++ b/src/pkg/fault/fault.go @@ -169,3 +169,7 @@ func (e *tracker) Add(err error) { func (e *tracker) Err() error { return e.current } + +// temporary hack identifier +// see: https://github.com/alcionai/corso/pull/2510#discussion_r1113532530 +const LabelForceNoBackupCreation = "label_forces_no_backup_creations"