diff --git a/src/internal/common/idname/idname.go b/src/internal/common/idname/idname.go index d56fab025..56460dd6e 100644 --- a/src/internal/common/idname/idname.go +++ b/src/internal/common/idname/idname.go @@ -40,6 +40,11 @@ type Cacher interface { ProviderForName(id string) Provider } +type CacheBuilder interface { + Add(id, name string) + Cacher +} + var _ Cacher = &cache{} type cache struct { @@ -47,17 +52,29 @@ type cache struct { nameToID map[string]string } -func NewCache(idToName map[string]string) cache { - nti := make(map[string]string, len(idToName)) - - for id, name := range idToName { - nti[name] = id +func NewCache(idToName map[string]string) *cache { + c := cache{ + idToName: map[string]string{}, + nameToID: map[string]string{}, } - return cache{ - idToName: idToName, - nameToID: nti, + if len(idToName) > 0 { + nti := make(map[string]string, len(idToName)) + + for id, name := range idToName { + nti[name] = id + } + + c.idToName = idToName + c.nameToID = nti } + + return &c +} + +func (c *cache) Add(id, name string) { + c.idToName[id] = name + c.nameToID[name] = id } // IDOf returns the id associated with the given name. diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index b0c8792e5..9b037350b 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -47,6 +48,11 @@ type Controller struct { // mutex used to synchronize updates to `status` mu sync.Mutex status support.ControllerOperationStatus // contains the status of the last run status + + // backupDriveIDNames is populated on restore. It maps the backup's + // drive names to their id. Primarily for use when creating or looking + // up a new drive. + backupDriveIDNames idname.CacheBuilder } func NewController( @@ -142,6 +148,20 @@ func (ctrl *Controller) incrementAwaitingMessages() { ctrl.wg.Add(1) } +func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { + if ctrl.backupDriveIDNames == nil { + ctrl.backupDriveIDNames = idname.NewCache(map[string]string{}) + } + + if dii.SharePoint != nil { + ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) + } + + if dii.OneDrive != nil { + ctrl.backupDriveIDNames.Add(dii.OneDrive.DriveID, dii.OneDrive.DriveName) + } +} + // --------------------------------------------------------------------------- // Resource Lookup Handling // --------------------------------------------------------------------------- diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 6d04b7e9e..48bd12ff3 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" @@ -22,6 +23,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" @@ -260,6 +262,82 @@ func (suite *ControllerUnitSuite) TestController_Wait() { assert.Equal(t, int64(4), result.Bytes) } +func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { + var ( + odid = "od-id" + odname = "od-name" + spid = "sp-id" + spname = "sp-name" + // intentionally declared outside the test loop + ctrl = &Controller{ + wg: &sync.WaitGroup{}, + region: &trace.Region{}, + backupDriveIDNames: idname.NewCache(nil), + } + ) + + table := []struct { + name string + service path.ServiceType + cat path.CategoryType + dii details.ItemInfo + expectID string + expectName string + }{ + { + name: "exchange", + dii: details.ItemInfo{ + Exchange: &details.ExchangeInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "folder", + dii: details.ItemInfo{ + Folder: &details.FolderInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "onedrive", + dii: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + DriveID: odid, + DriveName: odname, + }, + }, + expectID: odid, + expectName: odname, + }, + { + name: "sharepoint", + dii: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + DriveID: spid, + DriveName: spname, + }, + }, + expectID: spid, + expectName: spname, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctrl.CacheItemInfo(test.dii) + + name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID) + assert.Equal(t, test.expectName, name) + + id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName) + assert.Equal(t, test.expectID, id) + }) + } +} + // --------------------------------------------------------------------------- // Integration tests // --------------------------------------------------------------------------- diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 05cb8e159..977306883 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -69,3 +69,5 @@ func (ctrl Controller) ConsumeRestoreCollections( ) (*details.Details, error) { return ctrl.Deets, ctrl.Err } + +func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index 37cbd1c9b..ec2ab26af 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -361,7 +361,7 @@ func (suite *OneDriveIntgSuite) TestCreateGetDeleteFolder() { Folders: folderElements, } - caches := NewRestoreCaches() + caches := NewRestoreCaches(nil) caches.DriveIDToDriveInfo[driveID] = driveInfo{rootFolderID: ptr.Val(rootFolder.GetId())} rh := NewRestoreHandler(suite.ac) diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 4fcb07d84..58faa28cd 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -15,6 +15,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -44,6 +45,7 @@ type driveInfo struct { } type restoreCaches struct { + BackupDriveIDName idname.Cacher collisionKeyToItemID map[string]api.DriveItemIDType DriveIDToDriveInfo map[string]driveInfo DriveNameToDriveInfo map[string]driveInfo @@ -110,8 +112,16 @@ type GetDrivePagerAndRootFolderer interface { NewDrivePagerer } -func NewRestoreCaches() *restoreCaches { +func NewRestoreCaches( + backupDriveIDNames idname.Cacher, +) *restoreCaches { + // avoid nil panics + if backupDriveIDNames == nil { + backupDriveIDNames = idname.NewCache(nil) + } + return &restoreCaches{ + BackupDriveIDName: backupDriveIDNames, collisionKeyToItemID: map[string]api.DriveItemIDType{}, DriveIDToDriveInfo: map[string]driveInfo{}, DriveNameToDriveInfo: map[string]driveInfo{}, @@ -136,6 +146,7 @@ func ConsumeRestoreCollections( backupVersion int, restoreCfg control.RestoreConfig, opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, @@ -144,7 +155,7 @@ func ConsumeRestoreCollections( var ( restoreMetrics support.CollectionMetrics el = errs.Local() - caches = NewRestoreCaches() + caches = NewRestoreCaches(backupDriveIDNames) protectedResourceID = dcs[0].FullPath().ResourceOwner() fallbackDriveName = "" // onedrive cannot create drives ) diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index b2198d3a7..e8b41a45b 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -492,7 +492,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { mndi.SetId(ptr.To(mndiID)) var ( - caches = NewRestoreCaches() + caches = NewRestoreCaches(nil) rh = &mock.RestoreHandler{ PostItemResp: models.NewDriveItem(), DeleteItemErr: test.deleteErr, @@ -671,7 +671,7 @@ func (suite *RestoreUnitSuite) TestRestoreCaches_AddDrive() { ctx, flush := tester.NewContext(t) defer flush() - rc := NewRestoreCaches() + rc := NewRestoreCaches(nil) err := rc.AddDrive(ctx, md, test.mock) test.expectErr(t, err, clues.ToCore(err)) @@ -773,7 +773,7 @@ func (suite *RestoreUnitSuite) TestRestoreCaches_Populate() { pager: test.mock, } - rc := NewRestoreCaches() + rc := NewRestoreCaches(nil) err := rc.Populate(ctx, gdparf, "shmoo") test.expectErr(t, err, clues.ToCore(err)) @@ -849,7 +849,7 @@ func (suite *RestoreUnitSuite) TestEnsureDriveExists() { } populatedCache := func(id string) *restoreCaches { - rc := NewRestoreCaches() + rc := NewRestoreCaches(nil) di := driveInfo{ id: id, name: name, @@ -886,7 +886,7 @@ func (suite *RestoreUnitSuite) TestEnsureDriveExists() { postErr: []error{nil}, grf: grf, }, - rc: NewRestoreCaches(), + rc: NewRestoreCaches(nil), expectErr: require.NoError, expectName: name, }, @@ -897,7 +897,7 @@ func (suite *RestoreUnitSuite) TestEnsureDriveExists() { postErr: []error{assert.AnError}, grf: grf, }, - rc: NewRestoreCaches(), + rc: NewRestoreCaches(nil), expectErr: require.Error, expectName: "", skipValueChecks: true, @@ -920,7 +920,7 @@ func (suite *RestoreUnitSuite) TestEnsureDriveExists() { postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, grf: grf, }, - rc: NewRestoreCaches(), + rc: NewRestoreCaches(nil), expectErr: require.NoError, expectName: name + " 1", }, diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 3c5e3e646..d0ab3799b 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -54,6 +54,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( backupVersion, restoreCfg, opts, + ctrl.backupDriveIDNames, dcs, deets, errs, @@ -65,6 +66,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctrl.AC, restoreCfg, opts, + ctrl.backupDriveIDNames, dcs, deets, errs, diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index 9a18ebec1..c38b82e08 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -11,6 +11,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -34,6 +35,7 @@ func ConsumeRestoreCollections( ac api.Client, restoreCfg control.RestoreConfig, opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, @@ -43,7 +45,7 @@ func ConsumeRestoreCollections( lrh = libraryRestoreHandler{ac} protectedResourceID = dcs[0].FullPath().ResourceOwner() restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches() + caches = onedrive.NewRestoreCaches(backupDriveIDNames) el = errs.Local() ) diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index ae2c8d534..912b46743 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -46,6 +46,17 @@ type ( ) (*details.Details, error) Wait() *data.CollectionStats + + CacheItemInfoer + } + + CacheItemInfoer interface { + // CacheItemInfo is used by the consumer to cache metadata that is + // sourced from per-item info, but may be valuable to the restore at + // large. + // Ex: pairing drive ids with drive names as they appeared at the time + // of backup. + CacheItemInfo(v details.ItemInfo) } RepoMaintenancer interface { diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index e77b1104b..0f853a853 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -219,7 +219,13 @@ func (op *RestoreOperation) do( observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) - paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors) + paths, err := formatDetailsForRestoration( + ctx, + bup.Version, + op.Selectors, + deets, + op.rc, + op.Errors) if err != nil { return nil, clues.Wrap(err, "formatting paths from details") } @@ -359,6 +365,7 @@ func formatDetailsForRestoration( backupVersion int, sel selectors.Selector, deets *details.Details, + cii inject.CacheItemInfoer, errs *fault.Bus, ) ([]path.RestorePaths, error) { fds, err := sel.Reduce(ctx, deets, errs) @@ -366,6 +373,11 @@ func formatDetailsForRestoration( return nil, err } + // allow restore controllers to iterate over item metadata + for _, ent := range fds.Entries { + cii.CacheItemInfo(ent.ItemInfo) + } + paths, err := pathtransformer.GetPaths(ctx, backupVersion, fds.Items(), errs) if err != nil { return nil, clues.Wrap(err, "getting restore paths")