From 2cf44ee649c5eb0e5a3c19da2bc42d89786df15f Mon Sep 17 00:00:00 2001 From: Vaibhav Kamra Date: Thu, 5 Oct 2023 20:51:21 -0700 Subject: [PATCH] Dry-run support for Backup --- src/cli/backup/backup.go | 8 ++ src/cli/flags/options.go | 1 + src/cli/utils/options.go | 1 + src/cli/utils/utils.go | 5 ++ src/internal/data/metrics.go | 38 ++++++++ src/internal/m365/backup.go | 24 ++--- src/internal/m365/backup_test.go | 54 ++++++------ .../m365/collection/drive/collections.go | 31 ++++--- .../m365/collection/drive/collections_test.go | 2 +- .../collection/drive/item_collector_test.go | 2 +- .../m365/collection/exchange/backup.go | 40 ++++++--- src/internal/m365/collection/site/backup.go | 2 +- src/internal/m365/controller_test.go | 24 ++--- src/internal/m365/mock/connector.go | 6 +- src/internal/m365/service/exchange/backup.go | 31 +++++-- src/internal/m365/service/onedrive/backup.go | 13 +-- src/internal/operations/backup.go | 53 +++++++++-- src/internal/operations/inject/containers.go | 88 +++++++++++++++++++ src/internal/operations/inject/inject.go | 3 +- src/internal/operations/inject/mock/inject.go | 7 +- src/pkg/control/options.go | 1 + src/pkg/repository/repository.go | 3 +- 22 files changed, 328 insertions(+), 109 deletions(-) diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 5d885e059..e6db47210 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common/idname" @@ -214,6 +215,12 @@ func genericCreateCommand( continue } + if bo.Options.DryRun { + // Print backup stats results here + print.All(ctx, bo.Results.Stats) + continue + } + bIDs = append(bIDs, string(bo.Results.BackupID)) if !DisplayJSONFormat() { @@ -401,6 +408,7 @@ func printBackupStats(ctx context.Context, r repository.Repositoryer, bid string b, err := r.Backup(ctx, bid) if err != nil { logger.CtxErr(ctx, err).Error("finding backup immediately after backup operation completion") + return } b.ToPrintable().Stats.Print(ctx) diff --git a/src/cli/flags/options.go b/src/cli/flags/options.go index ba127092c..9035b3204 100644 --- a/src/cli/flags/options.go +++ b/src/cli/flags/options.go @@ -45,6 +45,7 @@ var ( // well-known flag values const ( RunModeFlagTest = "flag-test" + RunModeDryRun = "dry" RunModeRun = "run" ) diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index fdb92aced..992618430 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -28,6 +28,7 @@ func Control() control.Options { opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV opt.ToggleFeatures.DisableConcurrencyLimiter = flags.DisableConcurrencyLimiterFV opt.Parallelism.ItemFetch = flags.FetchParallelismFV + opt.DryRun = flags.RunModeFV == flags.RunModeDryRun return opt } diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 2ee9ac090..d9ebed273 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -68,6 +68,11 @@ func GetAccountAndConnectWithOverrides( opts := ControlWithConfig(cfg) + if opts.DryRun { + logger.CtxErr(ctx, err).Info("--dry-run is set") + opts.Repo.ReadOnly = true + } + r, err := repository.New( ctx, cfg.Account, diff --git a/src/internal/data/metrics.go b/src/internal/data/metrics.go index f34d20a16..7a86282d3 100644 --- a/src/internal/data/metrics.go +++ b/src/internal/data/metrics.go @@ -1,5 +1,12 @@ package data +import ( + "context" + "strconv" + + "github.com/alcionai/corso/src/cli/print" +) + type CollectionStats struct { Folders, Objects, @@ -15,3 +22,34 @@ func (cs CollectionStats) IsZero() bool { func (cs CollectionStats) String() string { return cs.Details } + +// interface compliance checks +var _ print.Printable = &CollectionStats{} + +// Print writes the Backup to StdOut, in the format requested by the caller. +func (cs CollectionStats) Print(ctx context.Context) { + print.Item(ctx, cs) +} + +// MinimumPrintable reduces the Backup to its minimally printable details. +func (cs CollectionStats) MinimumPrintable() any { + return cs +} + +// Headers returns the human-readable names of properties in a Backup +// for printing out to a terminal in a columnar display. +func (cs CollectionStats) Headers() []string { + return []string{ + "Folders", + "Objects", + } +} + +// Values returns the values matching the Headers list for printing +// out to a terminal in a columnar display. +func (cs CollectionStats) Values() []string { + return []string{ + strconv.Itoa(cs.Folders), + strconv.Itoa(cs.Objects), + } +} diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index 7b54c36cb..78921106e 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -36,7 +36,7 @@ func (ctrl *Controller) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, errs *fault.Bus, -) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) { +) (inject.BackupProducerResults, error) { service := bpc.Selector.PathService() ctx, end := diagnostics.Span( @@ -53,18 +53,19 @@ func (ctrl *Controller) ProduceBackupCollections( err := verifyBackupInputs(bpc.Selector, ctrl.IDNameLookup.IDs()) if err != nil { - return nil, nil, false, clues.Stack(err).WithClues(ctx) + return inject.BackupProducerResults{}, clues.Stack(err).WithClues(ctx) } var ( colls []data.BackupCollection ssmb *prefixmatcher.StringSetMatcher canUsePreviousBackup bool + results inject.BackupProducerResults ) switch service { case path.ExchangeService: - colls, ssmb, canUsePreviousBackup, err = exchange.ProduceBackupCollections( + results, err = exchange.ProduceBackupCollections( ctx, bpc, ctrl.AC, @@ -72,11 +73,11 @@ func (ctrl *Controller) ProduceBackupCollections( ctrl.UpdateStatus, errs) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } case path.OneDriveService: - colls, ssmb, canUsePreviousBackup, err = onedrive.ProduceBackupCollections( + results, err = onedrive.ProduceBackupCollections( ctx, bpc, ctrl.AC, @@ -84,7 +85,7 @@ func (ctrl *Controller) ProduceBackupCollections( ctrl.UpdateStatus, errs) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } case path.SharePointService: @@ -96,8 +97,9 @@ func (ctrl *Controller) ProduceBackupCollections( ctrl.UpdateStatus, errs) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } + results = inject.BackupProducerResults{Collections: colls, Excludes: ssmb, CanUsePreviousBackup: canUsePreviousBackup} case path.GroupsService: colls, ssmb, err = groups.ProduceBackupCollections( @@ -108,15 +110,15 @@ func (ctrl *Controller) ProduceBackupCollections( ctrl.UpdateStatus, errs) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } // canUsePreviousBacukp can be always returned true for groups as we // return a tombstone collection in case the metadata read fails canUsePreviousBackup = true - + results = inject.BackupProducerResults{Collections: colls, Excludes: ssmb, CanUsePreviousBackup: canUsePreviousBackup} default: - return nil, nil, false, clues.Wrap(clues.New(service.String()), "service not supported").WithClues(ctx) + return inject.BackupProducerResults{}, clues.Wrap(clues.New(service.String()), "service not supported").WithClues(ctx) } for _, c := range colls { @@ -131,7 +133,7 @@ func (ctrl *Controller) ProduceBackupCollections( } } - return colls, ssmb, canUsePreviousBackup, nil + return results, nil } func (ctrl *Controller) IsServiceEnabled( diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index f7e51f89d..9814e6f03 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -248,14 +248,14 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() ProtectedResource: test.getSelector(t), } - collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections( + results, err := ctrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) assert.Error(t, err, clues.ToCore(err)) - assert.False(t, canUsePreviousBackup, "can use previous backup") - assert.Empty(t, collections) - assert.Nil(t, excludes) + assert.False(t, results.CanUsePreviousBackup, "can use previous backup") + assert.Empty(t, results.Collections) + assert.Nil(t, results.Excludes) }) } } @@ -395,27 +395,27 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { Selector: sel.Selector, } - cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections( + results, err := ctrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") - require.Len(t, cols, 2) // 1 collection, 1 path prefix directory to ensure the root path exists. + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") + require.Len(t, results.Collections, 2) // 1 collection, 1 path prefix directory to ensure the root path exists. // No excludes yet as this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) - t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String()) + t.Logf("cols[0] Path: %s\n", results.Collections[0].FullPath().String()) assert.Equal( t, path.SharePointMetadataService.String(), - cols[0].FullPath().Service().String()) + results.Collections[0].FullPath().Service().String()) - t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String()) + t.Logf("cols[1] Path: %s\n", results.Collections[1].FullPath().String()) assert.Equal( t, path.SharePointService.String(), - cols[1].FullPath().Service().String()) + results.Collections[1].FullPath().Service().String()) } func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { @@ -445,17 +445,17 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { Selector: sel.Selector, } - cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections( + results, err := ctrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") - assert.Less(t, 0, len(cols)) + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") + assert.Less(t, 0, len(results.Collections)) // No excludes yet as this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) - for _, collection := range cols { + for _, collection := range results.Collections { t.Logf("Path: %s\n", collection.FullPath().String()) for item := range collection.Items(ctx, fault.New(true)) { @@ -531,18 +531,18 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint() Selector: sel.Selector, } - collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections( + results, err := ctrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") // No excludes yet as this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) // we don't know an exact count of drives this will produce, // but it should be more than one. - assert.Greater(t, len(collections), 1) + assert.Greater(t, len(results.Collections), 1) p, err := path.BuildMetadata( suite.tenantID, @@ -557,7 +557,7 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint() foundSitesMetadata := false - for _, coll := range collections { + for _, coll := range results.Collections { sitesMetadataCollection := coll.FullPath().String() == p.String() for object := range coll.Items(ctx, fault.New(true)) { @@ -631,18 +631,18 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint_In MetadataCollections: mmc, } - collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections( + results, err := ctrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") // No excludes yet as this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) // we don't know an exact count of drives this will produce, // but it should be more than one. - assert.Greater(t, len(collections), 1) + assert.Greater(t, len(results.Collections), 1) p, err := path.BuildMetadata( suite.tenantID, @@ -668,7 +668,7 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint_In sp, err = sp.Append(false, odConsts.SitesPathDir, ptr.Val(site.GetId())) require.NoError(t, err, clues.ToCore(err)) - for _, coll := range collections { + for _, coll := range results.Collections { if coll.State() == data.DeletedState { if coll.PreviousPath() != nil && coll.PreviousPath().String() == sp.String() { foundRootTombstone = true diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index 7d94156ea..7bbab4ea9 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -19,6 +19,7 @@ import ( odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" bupMD "github.com/alcionai/corso/src/pkg/backup/metadata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" @@ -226,10 +227,10 @@ func (c *Collections) Get( prevMetadata []data.RestoreCollection, ssmb *prefixmatcher.StringSetMatchBuilder, errs *fault.Bus, -) ([]data.BackupCollection, bool, error) { +) ([]data.BackupCollection, bool, inject.OneDriveStats, error) { prevDeltas, oldPathsByDriveID, canUsePreviousBackup, err := deserializeMetadata(ctx, prevMetadata) if err != nil { - return nil, false, err + return nil, false, inject.OneDriveStats{}, err } ctx = clues.Add(ctx, "can_use_previous_backup", canUsePreviousBackup) @@ -250,7 +251,7 @@ func (c *Collections) Get( drives, err := api.GetAllDrives(ctx, pager) if err != nil { - return nil, false, err + return nil, false, inject.OneDriveStats{}, err } var ( @@ -259,6 +260,7 @@ func (c *Collections) Get( // Drive ID -> folder ID -> folder path folderPaths = map[string]map[string]string{} numPrevItems = 0 + stats = inject.OneDriveStats{} ) for _, d := range drives { @@ -296,7 +298,7 @@ func (c *Collections) Get( prevDelta, errs) if err != nil { - return nil, false, err + return nil, false, inject.OneDriveStats{}, err } // Used for logging below. @@ -338,7 +340,7 @@ func (c *Collections) Get( prevDelta, errs) if err != nil { - return nil, false, err + return nil, false, inject.OneDriveStats{}, err } } @@ -351,7 +353,7 @@ func (c *Collections) Get( p, err := c.handler.CanonicalPath(odConsts.DriveFolderPrefixBuilder(driveID), c.tenantID) if err != nil { - return nil, false, clues.Wrap(err, "making exclude prefix").WithClues(ictx) + return nil, false, inject.OneDriveStats{}, clues.Wrap(err, "making exclude prefix").WithClues(ictx) } ssmb.Add(p.String(), excluded) @@ -379,7 +381,7 @@ func (c *Collections) Get( prevPath, err := path.FromDataLayerPath(p, false) if err != nil { err = clues.Wrap(err, "invalid previous path").WithClues(ictx).With("deleted_path", p) - return nil, false, err + return nil, false, inject.OneDriveStats{}, err } col, err := NewCollection( @@ -393,14 +395,19 @@ func (c *Collections) Get( true, nil) if err != nil { - return nil, false, clues.Wrap(err, "making collection").WithClues(ictx) + return nil, false, inject.OneDriveStats{}, clues.Wrap(err, "making collection").WithClues(ictx) } c.CollectionMap[driveID][fldID] = col } } + stats.Folders += c.NumContainers + stats.Items += c.NumFiles + observe.Message(ctx, fmt.Sprintf("Discovered %d items to backup", c.NumItems)) + observe.Message(ctx, fmt.Sprintf("Discovered %d stats to backup", stats.Items)) + observe.Message(ctx, fmt.Sprintf("Discovered %d folder stats to backup", stats.Folders)) collections := []data.BackupCollection{} @@ -415,7 +422,7 @@ func (c *Collections) Get( for driveID := range driveTombstones { prevDrivePath, err := c.handler.PathPrefix(c.tenantID, driveID) if err != nil { - return nil, false, clues.Wrap(err, "making drive tombstone for previous path").WithClues(ctx) + return nil, false, inject.OneDriveStats{}, clues.Wrap(err, "making drive tombstone for previous path").WithClues(ctx) } coll, err := NewCollection( @@ -429,7 +436,7 @@ func (c *Collections) Get( true, nil) if err != nil { - return nil, false, clues.Wrap(err, "making drive tombstone").WithClues(ctx) + return nil, false, inject.OneDriveStats{}, clues.Wrap(err, "making drive tombstone").WithClues(ctx) } collections = append(collections, coll) @@ -443,7 +450,7 @@ func (c *Collections) Get( // empty/missing and default to a full backup. logger.CtxErr(ctx, err).Info("making metadata collection path prefixes") - return collections, canUsePreviousBackup, nil + return collections, canUsePreviousBackup, inject.OneDriveStats{}, nil } md, err := graph.MakeMetadataCollection( @@ -463,7 +470,7 @@ func (c *Collections) Get( collections = append(collections, md) } - return collections, canUsePreviousBackup, nil + return collections, canUsePreviousBackup, stats, nil } // addURLCacheToDriveCollections adds an URL cache to all collections belonging to diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index 1e25d16c0..6f521376c 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -2306,7 +2306,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { delList := prefixmatcher.NewStringSetBuilder() - cols, canUsePreviousBackup, err := c.Get(ctx, prevMetadata, delList, errs) + cols, canUsePreviousBackup, _, err := c.Get(ctx, prevMetadata, delList, errs) test.errCheck(t, err) assert.Equal(t, test.canUsePreviousBackup, canUsePreviousBackup, "can use previous backup") assert.Equal(t, test.expectedSkippedCount, len(errs.Skipped())) diff --git a/src/internal/m365/collection/drive/item_collector_test.go b/src/internal/m365/collection/drive/item_collector_test.go index 0cc4d2a67..23b51c0a9 100644 --- a/src/internal/m365/collection/drive/item_collector_test.go +++ b/src/internal/m365/collection/drive/item_collector_test.go @@ -275,7 +275,7 @@ func (suite *OneDriveIntgSuite) TestOneDriveNewCollections() { ssmb := prefixmatcher.NewStringSetBuilder() - odcs, _, err := colls.Get(ctx, nil, ssmb, fault.New(true)) + odcs, _, _, err := colls.Get(ctx, nil, ssmb, fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) // Don't expect excludes as this isn't an incremental backup. assert.True(t, ssmb.Empty()) diff --git a/src/internal/m365/collection/exchange/backup.go b/src/internal/m365/collection/exchange/backup.go index 48193f89b..806d0bc40 100644 --- a/src/internal/m365/collection/exchange/backup.go +++ b/src/internal/m365/collection/exchange/backup.go @@ -33,7 +33,7 @@ func CreateCollections( dps metadata.DeltaPaths, su support.StatusUpdater, errs *fault.Bus, -) ([]data.BackupCollection, error) { +) ([]data.BackupCollection, inject.ExchangeStats, error) { ctx = clues.Add(ctx, "category", scope.Category().PathType()) var ( @@ -48,7 +48,7 @@ func CreateCollections( handler, ok := handlers[category] if !ok { - return nil, clues.New("unsupported backup category type").WithClues(ctx) + return nil, inject.ExchangeStats{}, clues.New("unsupported backup category type").WithClues(ctx) } foldersComplete := observe.MessageWithCompletion( @@ -59,10 +59,10 @@ func CreateCollections( rootFolder, cc := handler.NewContainerCache(bpc.ProtectedResource.ID()) if err := cc.Populate(ctx, errs, rootFolder); err != nil { - return nil, clues.Wrap(err, "populating container cache") + return nil, inject.ExchangeStats{}, clues.Wrap(err, "populating container cache") } - collections, err := populateCollections( + collections, stats, err := populateCollections( ctx, qp, handler, @@ -73,14 +73,14 @@ func CreateCollections( bpc.Options, errs) if err != nil { - return nil, clues.Wrap(err, "filling collections") + return nil, stats, clues.Wrap(err, "filling collections") } for _, coll := range collections { allCollections = append(allCollections, coll) } - return allCollections, nil + return allCollections, stats, nil } // populateCollections is a utility function @@ -102,7 +102,7 @@ func populateCollections( dps metadata.DeltaPaths, ctrlOpts control.Options, errs *fault.Bus, -) (map[string]data.BackupCollection, error) { +) (map[string]data.BackupCollection, inject.ExchangeStats, error) { var ( // folder ID -> BackupCollection. collections = map[string]data.BackupCollection{} @@ -113,6 +113,7 @@ func populateCollections( // deleted from this map, leaving only the deleted folders behind tombstones = makeTombstones(dps) category = qp.Category + stats = inject.ExchangeStats{} ) logger.Ctx(ctx).Infow("filling collections", "len_deltapaths", len(dps)) @@ -121,7 +122,7 @@ func populateCollections( for _, c := range resolver.Items() { if el.Failure() != nil { - return nil, el.Failure() + return nil, stats, el.Failure() } cID := ptr.Val(c.GetId()) @@ -209,6 +210,21 @@ func populateCollections( // add the current path for the container ID to be used in the next backup // as the "previous path", for reference in case of a rename or relocation. currPaths[cID] = currPath.String() + + switch category { + case path.EmailCategory: + stats.EmailFolders++ + stats.EmailsAdded += len(added) + stats.EmailsDeleted += len(removed) + case path.ContactsCategory: + stats.ContactFolders++ + stats.ContactsAdded += len(added) + stats.ContactsDeleted += len(removed) + case path.EventsCategory: + stats.EventFolders++ + stats.EventsAdded += len(added) + stats.EventsDeleted += len(removed) + } } // A tombstone is a folder that needs to be marked for deletion. @@ -217,7 +233,7 @@ func populateCollections( // resolver (which contains all the resource owners' current containers). for id, p := range tombstones { if el.Failure() != nil { - return nil, el.Failure() + return nil, stats, el.Failure() } var ( @@ -258,7 +274,7 @@ func populateCollections( qp.Category, false) if err != nil { - return nil, clues.Wrap(err, "making metadata path") + return nil, stats, clues.Wrap(err, "making metadata path") } col, err := graph.MakeMetadataCollection( @@ -269,12 +285,12 @@ func populateCollections( }, statusUpdater) if err != nil { - return nil, clues.Wrap(err, "making metadata collection") + return nil, stats, clues.Wrap(err, "making metadata collection") } collections["metadata"] = col - return collections, el.Failure() + return collections, stats, el.Failure() } // produces a set of id:path pairs from the deltapaths map. diff --git a/src/internal/m365/collection/site/backup.go b/src/internal/m365/collection/site/backup.go index 0ce62c14e..0a177beaa 100644 --- a/src/internal/m365/collection/site/backup.go +++ b/src/internal/m365/collection/site/backup.go @@ -43,7 +43,7 @@ func CollectLibraries( bpc.Options) ) - odcs, canUsePreviousBackup, err := colls.Get(ctx, bpc.MetadataCollections, ssmb, errs) + odcs, canUsePreviousBackup, _, err := colls.Get(ctx, bpc.MetadataCollections, ssmb, errs) if err != nil { return nil, false, graph.Wrap(ctx, err, "getting library") } diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index d95a56c9f..ce7d25994 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -600,14 +600,14 @@ func runBackupAndCompare( } start := time.Now() - dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( + results, err := backupCtrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") // No excludes yet because this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) t.Logf("Backup enumeration complete in %v\n", time.Since(start)) @@ -618,7 +618,7 @@ func runBackupAndCompare( ctx, totalKopiaItems, expectedData, - dcs, + results.Collections, sci) status := backupCtrl.Wait() @@ -1195,14 +1195,14 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { Selector: backupSel, } - dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( + results, err := backupCtrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") // No excludes yet because this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) t.Log("Backup enumeration complete") @@ -1217,7 +1217,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { // Pull the data prior to waiting for the status as otherwise it will // deadlock. - skipped := checkCollections(t, ctx, allItems, allExpectedData, dcs, ci) + skipped := checkCollections(t, ctx, allItems, allExpectedData, results.Collections, ci) status := backupCtrl.Wait() assert.Equal(t, allItems+skipped, status.Objects, "status.Objects") @@ -1374,20 +1374,20 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { Selector: backupSel, } - dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( + results, err := backupCtrl.ProduceBackupCollections( ctx, bpc, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - assert.True(t, canUsePreviousBackup, "can use previous backup") + assert.True(t, results.CanUsePreviousBackup, "can use previous backup") // No excludes yet because this isn't an incremental backup. - assert.True(t, excludes.Empty()) + assert.True(t, results.Excludes.Empty()) t.Logf("Backup enumeration complete in %v\n", time.Since(start)) // Use a map to find duplicates. foundCategories := []string{} - for _, col := range dcs { + for _, col := range results.Collections { // TODO(ashmrtn): We should be able to remove the below if we change how // status updates are done. Ideally we shouldn't have to fetch items in // these collections to avoid deadlocking. diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 20d17eed1..05cf5b556 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -42,12 +42,10 @@ func (ctrl Controller) ProduceBackupCollections( _ inject.BackupProducerConfig, _ *fault.Bus, ) ( - []data.BackupCollection, - prefixmatcher.StringSetReader, - bool, + inject.BackupProducerResults, error, ) { - return ctrl.Collections, ctrl.Exclude, ctrl.Err == nil, ctrl.Err + return inject.BackupProducerResults{Collections: ctrl.Collections, Excludes: ctrl.Exclude, CanUsePreviousBackup: ctrl.Err == nil}, ctrl.Err } func (ctrl *Controller) GetMetadataPaths( diff --git a/src/internal/m365/service/exchange/backup.go b/src/internal/m365/service/exchange/backup.go index 306125d2c..5dba18a96 100644 --- a/src/internal/m365/service/exchange/backup.go +++ b/src/internal/m365/service/exchange/backup.go @@ -5,7 +5,6 @@ import ( "github.com/alcionai/clues" - "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/exchange" "github.com/alcionai/corso/src/internal/m365/graph" @@ -26,22 +25,23 @@ func ProduceBackupCollections( tenantID string, su support.StatusUpdater, errs *fault.Bus, -) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { +) (inject.BackupProducerResults, error) { eb, err := bpc.Selector.ToExchangeBackup() if err != nil { - return nil, nil, false, clues.Wrap(err, "exchange dataCollection selector").WithClues(ctx) + return inject.BackupProducerResults{}, clues.Wrap(err, "exchange dataCollection selector").WithClues(ctx) } var ( collections = []data.BackupCollection{} el = errs.Local() categories = map[path.CategoryType]struct{}{} + mergedStats = inject.ExchangeStats{} handlers = exchange.BackupHandlers(ac) ) canMakeDeltaQueries, err := canMakeDeltaQueries(ctx, ac.Users(), bpc.ProtectedResource.ID()) if err != nil { - return nil, nil, false, clues.Stack(err) + return inject.BackupProducerResults{}, clues.Stack(err) } if !canMakeDeltaQueries { @@ -59,7 +59,7 @@ func ProduceBackupCollections( cdps, canUsePreviousBackup, err := exchange.ParseMetadataCollections(ctx, bpc.MetadataCollections) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } ctx = clues.Add(ctx, "can_use_previous_backup", canUsePreviousBackup) @@ -69,7 +69,7 @@ func ProduceBackupCollections( break } - dcs, err := exchange.CreateCollections( + dcs, stats, err := exchange.CreateCollections( ctx, bpc, handlers, @@ -86,6 +86,16 @@ func ProduceBackupCollections( categories[scope.Category().PathType()] = struct{}{} collections = append(collections, dcs...) + + mergedStats.ContactFolders += stats.ContactFolders + mergedStats.ContactsAdded += stats.ContactsAdded + mergedStats.ContactsDeleted += stats.ContactsDeleted + mergedStats.EventFolders += stats.EventFolders + mergedStats.EventsAdded += stats.EventsAdded + mergedStats.EventsDeleted += stats.EventsDeleted + mergedStats.EmailFolders += stats.EmailFolders + mergedStats.EmailsAdded += stats.EmailsAdded + mergedStats.EmailsDeleted += stats.EmailsDeleted } if len(collections) > 0 { @@ -99,13 +109,18 @@ func ProduceBackupCollections( su, errs) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } collections = append(collections, baseCols...) } - return collections, nil, canUsePreviousBackup, el.Failure() + return inject.BackupProducerResults{ + Collections: collections, + Excludes: nil, + CanUsePreviousBackup: canUsePreviousBackup, + DiscoveredItems: inject.Stats{Exchange: &mergedStats}}, + el.Failure() } func canMakeDeltaQueries( diff --git a/src/internal/m365/service/onedrive/backup.go b/src/internal/m365/service/onedrive/backup.go index b94ce918d..61b7e1c78 100644 --- a/src/internal/m365/service/onedrive/backup.go +++ b/src/internal/m365/service/onedrive/backup.go @@ -25,10 +25,10 @@ func ProduceBackupCollections( tenant string, su support.StatusUpdater, errs *fault.Bus, -) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { +) (inject.BackupProducerResults, error) { odb, err := bpc.Selector.ToOneDriveBackup() if err != nil { - return nil, nil, false, clues.Wrap(err, "parsing selector").WithClues(ctx) + return inject.BackupProducerResults{}, clues.Wrap(err, "parsing selector").WithClues(ctx) } var ( @@ -38,6 +38,7 @@ func ProduceBackupCollections( ssmb = prefixmatcher.NewStringSetBuilder() odcs []data.BackupCollection canUsePreviousBackup bool + stats = inject.OneDriveStats{} ) // for each scope that includes oneDrive items, get all @@ -55,7 +56,7 @@ func ProduceBackupCollections( su, bpc.Options) - odcs, canUsePreviousBackup, err = nc.Get(ctx, bpc.MetadataCollections, ssmb, errs) + odcs, canUsePreviousBackup, stats, err = nc.Get(ctx, bpc.MetadataCollections, ssmb, errs) if err != nil { el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation)) } @@ -67,7 +68,7 @@ func ProduceBackupCollections( mcs, err := migrationCollections(bpc, tenant, su) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } collections = append(collections, mcs...) @@ -83,13 +84,13 @@ func ProduceBackupCollections( su, errs) if err != nil { - return nil, nil, false, err + return inject.BackupProducerResults{}, err } collections = append(collections, baseCols...) } - return collections, ssmb.ToReader(), canUsePreviousBackup, el.Failure() + return inject.BackupProducerResults{Collections: collections, Excludes: ssmb.ToReader(), CanUsePreviousBackup: canUsePreviousBackup, DiscoveredItems: inject.Stats{OneDrive: &stats}}, el.Failure() } // adds data migrations to the collection set. diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 80b74277e..2801992f1 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -67,6 +67,7 @@ type BackupOperation struct { type BackupResults struct { stats.ReadWrites stats.StartAndEndTime + inject.Stats BackupID model.StableID `json:"backupID"` } @@ -291,6 +292,11 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { LogFaultErrors(ctx, op.Errors.Errors(), "running backup") + // Don't persist any results for a dry-run operation + if op.Options.DryRun { + return nil + } + // ----- // Persistence // ----- @@ -382,7 +388,7 @@ func (op *BackupOperation) do( // the entire subtree instead of returning an additional bool. That way base // selection is controlled completely by flags and merging is controlled // completely by collections. - cs, ssmb, canUsePreviousBackup, err := produceBackupDataCollections( + producerResults, err := produceBackupDataCollections( ctx, op.bp, op.ResourceOwner, @@ -397,8 +403,14 @@ func (op *BackupOperation) do( ctx = clues.Add( ctx, - "can_use_previous_backup", canUsePreviousBackup, - "collection_count", len(cs)) + "can_use_previous_backup", producerResults.CanUsePreviousBackup, + "collection_count", len(producerResults.Collections)) + + // Do nothing with the data collections + if op.Options.DryRun { + op.Results.Stats = producerResults.DiscoveredItems + return nil, nil + } writeStats, deets, toMerge, err := consumeBackupCollections( ctx, @@ -406,10 +418,10 @@ func (op *BackupOperation) do( op.account.ID(), reasons, mans, - cs, - ssmb, + producerResults.Collections, + producerResults.Excludes, backupID, - op.incremental && canUseMetadata && canUsePreviousBackup, + op.incremental && canUseMetadata && producerResults.CanUsePreviousBackup, op.Errors) if err != nil { return nil, clues.Wrap(err, "persisting collection backups") @@ -439,6 +451,33 @@ func (op *BackupOperation) do( return deets, nil } +// func summarizeBackupCollections(ctx context.Context, results inject.BackupProducerResults) (stats.BackupItems, error) { +// collStats := stats.BackupItems{} +// bus := fault.New(false) + +// for _, c := range cs { +// switch c.State() { +// case data.NewState: +// logger.Ctx(ctx).Infow("New Folder:", c.FullPath()) +// collStats.NewFolders++ +// case data.NotMovedState, data.MovedState: +// logger.Ctx(ctx).Infow("Modified Folder:", c.FullPath()) +// collStats.ModifiedFolders++ +// case data.DeletedState: +// logger.Ctx(ctx).Infow("Deleted Folder:", c.FullPath()) +// collStats.DeletedFolders++ +// } + +// for i := range c.Items(ctx, bus) { +// logger.Ctx(ctx).Infow("Item", i.ID()) +// collStats.Items++ +// } +// } + +// return collStats, nil + +// } + func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) { if sel.PathService() != path.SharePointService && sel.DiscreteOwner != sel.DiscreteOwnerName { @@ -469,7 +508,7 @@ func produceBackupDataCollections( lastBackupVersion int, ctrlOpts control.Options, errs *fault.Bus, -) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) { +) (inject.BackupProducerResults, error) { progressBar := observe.MessageWithCompletion(ctx, "Discovering items to backup") defer close(progressBar) diff --git a/src/internal/operations/inject/containers.go b/src/internal/operations/inject/containers.go index 8594c813b..baba6616e 100644 --- a/src/internal/operations/inject/containers.go +++ b/src/internal/operations/inject/containers.go @@ -1,7 +1,12 @@ package inject import ( + "context" + "strconv" + + "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/selectors" @@ -28,3 +33,86 @@ type BackupProducerConfig struct { ProtectedResource idname.Provider Selector selectors.Selector } + +type BackupProducerResults struct { + Collections []data.BackupCollection + Excludes prefixmatcher.StringSetReader + CanUsePreviousBackup bool + DiscoveredItems Stats +} + +// Stats is a oneOf that contains service specific +// information +type Stats struct { + Exchange *ExchangeStats `json:"exchange,omitempty"` + SharePoint *SharePointStats `json:"sharePoint,omitempty"` + OneDrive *OneDriveStats `json:"oneDrive,omitempty"` + Groups *GroupsStats `json:"groups,omitempty"` +} + +type ExchangeStats struct { + ContactsAdded int `json:"contactsAdded,omitempty"` + ContactsDeleted int `json:"contactsDeleted,omitempty"` + ContactFolders int `json:"contactFolders,omitempty"` + + EventsAdded int `json:"eventsAdded,omitempty"` + EventsDeleted int `json:"eventsDeleted,omitempty"` + EventFolders int `json:"eventFolders,omitempty"` + + EmailsAdded int `json:"emailsAdded,omitempty"` + EmailsDeleted int `json:"emailsDeleted,omitempty"` + EmailFolders int `json:"emailFolders,omitempty"` +} + +type SharePointStats struct { +} +type OneDriveStats struct { + Folders int `json:"folders,omitempty"` + Items int `json:"items,omitempty"` +} +type GroupsStats struct { +} + +// interface compliance checks +var _ print.Printable = &Stats{} + +// Print writes the Backup to StdOut, in the format requested by the caller. +func (s Stats) Print(ctx context.Context) { + print.Item(ctx, s) +} + +// MinimumPrintable reduces the Backup to its minimally printable details. +func (s Stats) MinimumPrintable() any { + return s +} + +// Headers returns the human-readable names of properties in a Backup +// for printing out to a terminal in a columnar display. +func (s Stats) Headers() []string { + switch { + case s.OneDrive != nil: + return []string{ + "Folders", + "Items", + } + + default: + return []string{} + } +} + +// Values returns the values matching the Headers list for printing +// out to a terminal in a columnar display. +func (s Stats) Values() []string { + switch { + case s.OneDrive != nil: + return []string{ + strconv.Itoa(s.OneDrive.Folders), + strconv.Itoa(s.OneDrive.Items), + } + + default: + return []string{} + + } +} diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index e7b4ba228..6a3cc5f57 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -4,7 +4,6 @@ import ( "context" "github.com/alcionai/corso/src/internal/common/idname" - "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia/inject" @@ -24,7 +23,7 @@ type ( ctx context.Context, bpc BackupProducerConfig, errs *fault.Bus, - ) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) + ) (BackupProducerResults, error) IsServiceEnableder diff --git a/src/internal/operations/inject/mock/inject.go b/src/internal/operations/inject/mock/inject.go index 408da22b9..96daf0681 100644 --- a/src/internal/operations/inject/mock/inject.go +++ b/src/internal/operations/inject/mock/inject.go @@ -5,7 +5,6 @@ import ( "github.com/alcionai/clues" - "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/kopia" kinject "github.com/alcionai/corso/src/internal/kopia/inject" @@ -39,12 +38,12 @@ func (mbp *mockBackupProducer) ProduceBackupCollections( context.Context, inject.BackupProducerConfig, *fault.Bus, -) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) { +) (inject.BackupProducerResults, error) { if mbp.injectNonRecoverableErr { - return nil, nil, false, clues.New("non-recoverable error") + return inject.BackupProducerResults{}, clues.New("non-recoverable error") } - return mbp.colls, nil, true, nil + return inject.BackupProducerResults{Collections: mbp.colls, Excludes: nil, CanUsePreviousBackup: true}, nil } func (mbp *mockBackupProducer) IsServiceEnabled( diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index 0f7d559aa..51eb8881b 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -17,6 +17,7 @@ type Options struct { Repo repository.Options `json:"repo"` SkipReduce bool `json:"skipReduce"` ToggleFeatures Toggles `json:"toggleFeatures"` + DryRun bool `json:"dryRun"` } type Parallelism struct { diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 283af8e56..ee8b5600e 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -192,7 +192,8 @@ type ConnConfig struct { // tells the data provider which service to // use for its connection pattern. Leave empty // to skip the provider connection. - Service path.ServiceType + Service path.ServiceType + ReadOnly bool } // Connect will: