diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index f1b863275..d1855b108 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -18,7 +18,9 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" + m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" @@ -84,14 +86,14 @@ func generateAndRestoreItems( items: items, }} - dest := control.DefaultRestoreConfig(dttm.SafeForTesting) - dest.Location = destFldr - print.Infof(ctx, "Restoring to folder %s", dest.Location) + restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) + restoreCfg.Location = destFldr + print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location) dataColls, err := buildCollections( service, tenantID, userID, - dest, + restoreCfg, collections) if err != nil { return nil, err @@ -99,7 +101,7 @@ func generateAndRestoreItems( print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, dataColls, errs) + return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs) } // ------------------------------------------------------------------------------------------ @@ -108,7 +110,7 @@ func generateAndRestoreItems( func getControllerAndVerifyResourceOwner( ctx context.Context, - rc resource.Category, + resourceCat resource.Category, resourceOwner string, ) ( *m365.Controller, @@ -133,7 +135,7 @@ func getControllerAndVerifyResourceOwner( return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details") } - ctrl, err := m365.NewController(ctx, acct, rc) + ctrl, err := m365.NewController(ctx, acct, resourceCat) if err != nil { return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") } @@ -164,7 +166,7 @@ type collection struct { func buildCollections( service path.ServiceType, tenant, user string, - dest control.RestoreConfig, + restoreCfg control.RestoreConfig, colls []collection, ) ([]data.RestoreCollection, error) { collections := make([]data.RestoreCollection, 0, len(colls)) @@ -225,9 +227,9 @@ func generateAndRestoreDriveItems( ctx, flush := tester.NewContext(nil) defer flush() - dest := control.DefaultRestoreConfig(dttm.SafeForTesting) - dest.Location = destFldr - print.Infof(ctx, "Restoring to folder %s", dest.Location) + restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) + restoreCfg.Location = destFldr + print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location) var driveID string @@ -249,7 +251,7 @@ func generateAndRestoreDriveItems( } var ( - cols []m365.OnedriveColInfo + cols []odStub.ColInfo rootPath = []string{"drives", driveID, "root:"} folderAPath = []string{"drives", driveID, "root:", folderAName} @@ -263,15 +265,15 @@ func generateAndRestoreDriveItems( ) for i := 0; i < count; i++ { - col := []m365.OnedriveColInfo{ + col := []odStub.ColInfo{ // basic folder and file creation { PathElements: rootPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime), Data: fileAData, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -282,13 +284,13 @@ func generateAndRestoreDriveItems( Data: fileBData, }, }, - Folders: []m365.ItemData{ + Folders: []odStub.ItemData{ { Name: folderBName, }, { Name: folderAName, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -296,7 +298,7 @@ func generateAndRestoreDriveItems( }, { Name: folderCName, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -308,18 +310,18 @@ func generateAndRestoreDriveItems( // a folder that has permissions with an item in the folder with // the different permissions. PathElements: folderAPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Data: fileEData, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -329,13 +331,13 @@ func generateAndRestoreDriveItems( // a folder that has permissions with an item in the folder with // no permissions. PathElements: folderCPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Data: fileAData, }, }, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -343,23 +345,23 @@ func generateAndRestoreDriveItems( }, { PathElements: folderBPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { // restoring a file in a non-root folder that doesn't inherit // permissions. Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Data: fileBData, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Folders: []m365.ItemData{ + Folders: []odStub.ItemData{ { Name: folderAName, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -372,7 +374,7 @@ func generateAndRestoreDriveItems( cols = append(cols, col...) } - input, err := m365.DataForInfo(service, cols, version.Backup) + input, err := odStub.DataForInfo(service, cols, version.Backup) if err != nil { return nil, err } @@ -389,7 +391,7 @@ func generateAndRestoreDriveItems( ToggleFeatures: control.Toggles{}, } - config := m365.ConfigInfo{ + config := m365Stub.ConfigInfo{ Opts: opts, Resource: resource.Users, Service: service, @@ -398,7 +400,7 @@ func generateAndRestoreDriveItems( RestoreCfg: tester.DefaultTestRestoreConfig(""), } - _, _, collections, _, err := m365.GetCollectionsAndExpected( + _, _, collections, _, err := m365Stub.GetCollectionsAndExpected( config, input, version.Backup) @@ -406,5 +408,5 @@ func generateAndRestoreDriveItems( return nil, err } - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, collections, errs) + return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs) } diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 9429be012..03543061b 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -66,7 +66,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { selUsers := []string{suite.user} - ctrl := loadController(ctx, suite.T(), resource.Users) + ctrl := newController(ctx, suite.T(), resource.Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -167,7 +167,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() defer flush() owners := []string{"snuffleupagus"} - ctrl := loadController(ctx, suite.T(), resource.Users) + ctrl := newController(ctx, suite.T(), resource.Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -253,7 +253,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { defer flush() selSites := []string{suite.site} - ctrl := loadController(ctx, suite.T(), resource.Sites) + ctrl := newController(ctx, suite.T(), resource.Sites) tests := []struct { name string expected int @@ -348,7 +348,7 @@ func (suite *SPCollectionIntgSuite) SetupSuite() { ctx, flush := tester.NewContext(suite.T()) defer flush() - suite.connector = loadController(ctx, suite.T(), resource.Sites) + suite.connector = newController(ctx, suite.T(), resource.Sites) suite.user = tester.M365UserID(suite.T()) tester.LogTimeOfTest(suite.T()) @@ -362,7 +362,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { var ( siteID = tester.M365SiteID(t) - ctrl = loadController(ctx, t, resource.Sites) + ctrl = newController(ctx, t, resource.Sites) siteIDs = []string{siteID} ) @@ -409,7 +409,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { var ( siteID = tester.M365SiteID(t) - ctrl = loadController(ctx, t, resource.Sites) + ctrl = newController(ctx, t, resource.Sites) siteIDs = []string{siteID} ) diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index e7be0ae5f..2d63e4261 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -17,6 +17,7 @@ import ( exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" + "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -24,6 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" ) // --------------------------------------------------------------------------- @@ -280,7 +282,7 @@ func (suite *ControllerIntegrationSuite) SetupSuite() { ctx, flush := tester.NewContext(t) defer flush() - suite.ctrl = loadController(ctx, t, resource.Users) + suite.ctrl = newController(ctx, t, resource.Users) suite.user = tester.M365UserID(t) suite.secondaryUser = tester.SecondaryM365UserID(t) @@ -407,7 +409,7 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { func runRestore( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument - config ConfigInfo, + config stub.ConfigInfo, backupVersion int, collections []data.RestoreCollection, numRestoreItems int, @@ -419,7 +421,7 @@ func runRestore( start := time.Now() - restoreCtrl := loadController(ctx, t, config.Resource) + restoreCtrl := newController(ctx, t, config.Resource) restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true) deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, @@ -450,11 +452,11 @@ func runRestore( func runBackupAndCompare( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument - config ConfigInfo, + config stub.ConfigInfo, expectedData map[string]map[string][]byte, totalItems int, totalKopiaItems int, - inputCollections []ColInfo, + inputCollections []stub.ColInfo, ) { t.Helper() @@ -481,7 +483,7 @@ func runBackupAndCompare( nameToID[ro] = ro } - backupCtrl := loadController(ctx, t, config.Resource) + backupCtrl := newController(ctx, t, config.Resource) backupCtrl.IDNameLookup = inMock.NewCache(idToName, nameToID) backupSel := backupSelectorForExpected(t, config.Service, expectedDests) @@ -531,7 +533,7 @@ func runRestoreBackupTest( ctx, flush := tester.NewContext(t) defer flush() - config := ConfigInfo{ + config := stub.ConfigInfo{ Opts: opts, Resource: test.resourceCat, Service: test.service, @@ -540,7 +542,7 @@ func runRestoreBackupTest( RestoreCfg: tester.DefaultTestRestoreConfig(""), } - totalItems, totalKopiaItems, collections, expectedData, err := GetCollectionsAndExpected( + totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected( config, test.collections, version.Backup) @@ -576,16 +578,16 @@ func runRestoreTestWithVersion( ctx, flush := tester.NewContext(t) defer flush() - config := ConfigInfo{ + config := stub.ConfigInfo{ Opts: opts, - Resource: test.resource, + Resource: test.resourceCat, Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, RestoreCfg: tester.DefaultTestRestoreConfig(""), } - totalItems, _, collections, _, err := GetCollectionsAndExpected( + totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( config, test.collectionsPrevious, test.backupVersion) @@ -613,16 +615,16 @@ func runRestoreBackupTestVersions( ctx, flush := tester.NewContext(t) defer flush() - config := ConfigInfo{ + config := stub.ConfigInfo{ Opts: opts, - Resource: test.resource, + Resource: test.resourceCat, Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, RestoreCfg: tester.DefaultTestRestoreConfig(""), } - totalItems, _, collections, _, err := GetCollectionsAndExpected( + totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( config, test.collectionsPrevious, test.backupVersion) @@ -637,7 +639,7 @@ func runRestoreBackupTestVersions( totalItems) // Get expected output for new version. - totalItems, totalKopiaItems, _, expectedData, err := GetCollectionsAndExpected( + totalItems, totalKopiaItems, _, expectedData, err := stub.GetCollectionsAndExpected( config, test.collectionsLatest, version.Backup) @@ -662,24 +664,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "EmailsWithAttachments", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.MessageWithDirectAttachment( + Name: "someencodeditemID", + Data: exchMock.MessageWithDirectAttachment( subjectText + "-1", ), - lookupKey: subjectText + "-1", + LookupKey: subjectText + "-1", }, { - name: "someencodeditemID2", - data: exchMock.MessageWithTwoAttachments( + Name: "someencodeditemID2", + Data: exchMock.MessageWithTwoAttachments( subjectText + "-2", ), - lookupKey: subjectText + "-2", + LookupKey: subjectText + "-2", }, }, }, @@ -689,73 +691,73 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "MultipleEmailsMultipleFolders", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID", + Data: exchMock.MessageWithBodyBytes( subjectText+"-1", bodyText+" 1.", bodyText+" 1.", ), - lookupKey: subjectText + "-1", + LookupKey: subjectText + "-1", }, }, }, { PathElements: []string{"Work"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID2", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID2", + Data: exchMock.MessageWithBodyBytes( subjectText+"-2", bodyText+" 2.", bodyText+" 2.", ), - lookupKey: subjectText + "-2", + LookupKey: subjectText + "-2", }, { - name: "someencodeditemID3", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID3", + Data: exchMock.MessageWithBodyBytes( subjectText+"-3", bodyText+" 3.", bodyText+" 3.", ), - lookupKey: subjectText + "-3", + LookupKey: subjectText + "-3", }, }, }, { PathElements: []string{"Work", "Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID4", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID4", + Data: exchMock.MessageWithBodyBytes( subjectText+"-4", bodyText+" 4.", bodyText+" 4.", ), - lookupKey: subjectText + "-4", + LookupKey: subjectText + "-4", }, }, }, { PathElements: []string{"Work", "Inbox", "Work"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID5", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID5", + Data: exchMock.MessageWithBodyBytes( subjectText+"-5", bodyText+" 5.", bodyText+" 5.", ), - lookupKey: subjectText + "-5", + LookupKey: subjectText + "-5", }, }, }, @@ -765,25 +767,25 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "MultipleContactsSingleFolder", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Contacts"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.ContactBytes("Ghimley"), - lookupKey: "Ghimley", + Name: "someencodeditemID", + Data: exchMock.ContactBytes("Ghimley"), + LookupKey: "Ghimley", }, { - name: "someencodeditemID2", - data: exchMock.ContactBytes("Irgot"), - lookupKey: "Irgot", + Name: "someencodeditemID2", + Data: exchMock.ContactBytes("Irgot"), + LookupKey: "Irgot", }, { - name: "someencodeditemID3", - data: exchMock.ContactBytes("Jannes"), - lookupKey: "Jannes", + Name: "someencodeditemID3", + Data: exchMock.ContactBytes("Jannes"), + LookupKey: "Jannes", }, }, }, @@ -793,41 +795,41 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "MultipleContactsMultipleFolders", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Work"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.ContactBytes("Ghimley"), - lookupKey: "Ghimley", + Name: "someencodeditemID", + Data: exchMock.ContactBytes("Ghimley"), + LookupKey: "Ghimley", }, { - name: "someencodeditemID2", - data: exchMock.ContactBytes("Irgot"), - lookupKey: "Irgot", + Name: "someencodeditemID2", + Data: exchMock.ContactBytes("Irgot"), + LookupKey: "Irgot", }, { - name: "someencodeditemID3", - data: exchMock.ContactBytes("Jannes"), - lookupKey: "Jannes", + Name: "someencodeditemID3", + Data: exchMock.ContactBytes("Jannes"), + LookupKey: "Jannes", }, }, }, { PathElements: []string{"Personal"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID4", - data: exchMock.ContactBytes("Argon"), - lookupKey: "Argon", + Name: "someencodeditemID4", + Data: exchMock.ContactBytes("Argon"), + LookupKey: "Argon", }, { - name: "someencodeditemID5", - data: exchMock.ContactBytes("Bernard"), - lookupKey: "Bernard", + Name: "someencodeditemID5", + Data: exchMock.ContactBytes("Bernard"), + LookupKey: "Bernard", }, }, }, @@ -926,26 +928,26 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { name: "Contacts", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Work"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.ContactBytes("Ghimley"), - lookupKey: "Ghimley", + Name: "someencodeditemID", + Data: exchMock.ContactBytes("Ghimley"), + LookupKey: "Ghimley", }, }, }, { PathElements: []string{"Personal"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID2", - data: exchMock.ContactBytes("Irgot"), - lookupKey: "Irgot", + Name: "someencodeditemID2", + Data: exchMock.ContactBytes("Irgot"), + LookupKey: "Irgot", }, }, }, @@ -1004,12 +1006,12 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { }, }) - totalItems, _, collections, expectedData, err := collectionsForInfo( + totalItems, _, collections, expectedData, err := stub.CollectionsForInfo( test.service, suite.ctrl.tenant, suite.user, restoreCfg, - []ColInfo{collection}, + []stub.ColInfo{collection}, version.Backup, ) require.NoError(t, err) @@ -1027,7 +1029,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { restoreCfg.Location, ) - restoreCtrl := loadController(ctx, t, test.resourceCat) + restoreCtrl := newController(ctx, t, test.resourceCat) deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, version.Backup, @@ -1057,7 +1059,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { // Run a backup and compare its output with what we put in. - backupCtrl := loadController(ctx, t, test.resourceCat) + backupCtrl := newController(ctx, t, test.resourceCat) backupSel := backupSelectorForExpected(t, test.service, expectedDests) t.Log("Selective backup of", backupSel) @@ -1079,7 +1081,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { t.Log("Backup enumeration complete") - ci := ConfigInfo{ + ci := stub.ConfigInfo{ Opts: control.Options{RestorePermissions: true}, // Alright to be empty, needed for OneDrive. RestoreCfg: control.RestoreConfig{}, @@ -1105,15 +1107,15 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen name: "EmailsWithLargeAttachments", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "35mbAttachment", - data: exchMock.MessageWithSizedAttachment(subjectText, 35), - lookupKey: subjectText, + Name: "35mbAttachment", + Data: exchMock.MessageWithSizedAttachment(subjectText, 35), + LookupKey: subjectText, }, }, }, @@ -1206,7 +1208,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { defer flush() var ( - backupCtrl = loadController(ctx, t, test.resourceCat) + backupCtrl = newController(ctx, t, test.resourceCat) backupSel = test.selectorFunc(t) errs = fault.New(true) start = time.Now() @@ -1270,3 +1272,166 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { }) } } + +type DisconnectedUnitSuite struct { + tester.Suite +} + +func TestDisconnectedUnitSuite(t *testing.T) { + s := &DisconnectedUnitSuite{ + Suite: tester.NewUnitSuite(t), + } + + suite.Run(t, s) +} + +func statusTestTask( + t *testing.T, + ctrl *Controller, + objects, success, folder int, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + status := support.CreateStatus( + ctx, + support.Restore, folder, + support.CollectionMetrics{ + Objects: objects, + Successes: success, + Bytes: 0, + }, + "statusTestTask") + ctrl.UpdateStatus(status) +} + +func (suite *DisconnectedUnitSuite) TestController_Status() { + t := suite.T() + ctrl := Controller{wg: &sync.WaitGroup{}} + + // Two tasks + ctrl.incrementAwaitingMessages() + ctrl.incrementAwaitingMessages() + + // Each helper task processes 4 objects, 1 success, 3 errors, 1 folders + go statusTestTask(t, &ctrl, 4, 1, 1) + go statusTestTask(t, &ctrl, 4, 1, 1) + + stats := ctrl.Wait() + + assert.NotEmpty(t, ctrl.PrintableStatus()) + // Expect 8 objects + assert.Equal(t, 8, stats.Objects) + // Expect 2 success + assert.Equal(t, 2, stats.Successes) + // Expect 2 folders + assert.Equal(t, 2, stats.Folders) +} + +func (suite *DisconnectedUnitSuite) TestVerifyBackupInputs_allServices() { + sites := []string{"abc.site.foo", "bar.site.baz"} + + tests := []struct { + name string + excludes func(t *testing.T) selectors.Selector + filters func(t *testing.T) selectors.Selector + includes func(t *testing.T) selectors.Selector + checkError assert.ErrorAssertionFunc + }{ + { + name: "Valid User", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = "elliotReid@someHospital.org" + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = "elliotReid@someHospital.org" + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = "elliotReid@someHospital.org" + return sel.Selector + }, + }, + { + name: "Invalid User", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + return sel.Selector + }, + }, + { + name: "valid sites", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) + sel.DiscreteOwner = "abc.site.foo" + sel.Exclude(sel.AllData()) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) + sel.DiscreteOwner = "abc.site.foo" + sel.Filter(sel.AllData()) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) + sel.DiscreteOwner = "abc.site.foo" + sel.Include(sel.AllData()) + return sel.Selector + }, + }, + { + name: "invalid sites", + checkError: assert.Error, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) + sel.Exclude(sel.AllData()) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) + sel.Filter(sel.AllData()) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) + sel.Include(sel.AllData()) + return sel.Selector + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + err := verifyBackupInputs(test.excludes(t), sites) + test.checkError(t, err, clues.ToCore(err)) + err = verifyBackupInputs(test.filters(t), sites) + test.checkError(t, err, clues.ToCore(err)) + err = verifyBackupInputs(test.includes(t), sites) + test.checkError(t, err, clues.ToCore(err)) + }) + } +} diff --git a/src/internal/m365/exchange/contact_container_cache.go b/src/internal/m365/exchange/contacts_container_cache.go similarity index 100% rename from src/internal/m365/exchange/contact_container_cache.go rename to src/internal/m365/exchange/contacts_container_cache.go diff --git a/src/internal/m365/exchange/event_container_cache.go b/src/internal/m365/exchange/events_container_cache.go similarity index 100% rename from src/internal/m365/exchange/event_container_cache.go rename to src/internal/m365/exchange/events_container_cache.go diff --git a/src/internal/m365/graph_connector_disconnected_test.go b/src/internal/m365/graph_connector_disconnected_test.go deleted file mode 100644 index c2dee9d58..000000000 --- a/src/internal/m365/graph_connector_disconnected_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package m365 - -import ( - "sync" - "testing" - - "github.com/alcionai/clues" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/internal/m365/support" - "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/selectors" - selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" -) - -// --------------------------------------------------------------- -// Disconnected Test Section -// --------------------------------------------------------------- -type DisconnectedSuite struct { - tester.Suite -} - -func TestSuite(t *testing.T) { - s := &DisconnectedSuite{ - Suite: tester.NewUnitSuite(t), - } - - suite.Run(t, s) -} - -func statusTestTask( - t *testing.T, - ctrl *Controller, - objects, success, folder int, -) { - ctx, flush := tester.NewContext(t) - defer flush() - - status := support.CreateStatus( - ctx, - support.Restore, folder, - support.CollectionMetrics{ - Objects: objects, - Successes: success, - Bytes: 0, - }, - "statusTestTask") - ctrl.UpdateStatus(status) -} - -func (suite *DisconnectedSuite) TestController_Status() { - t := suite.T() - ctrl := Controller{wg: &sync.WaitGroup{}} - - // Two tasks - ctrl.incrementAwaitingMessages() - ctrl.incrementAwaitingMessages() - - // Each helper task processes 4 objects, 1 success, 3 errors, 1 folders - go statusTestTask(t, &ctrl, 4, 1, 1) - go statusTestTask(t, &ctrl, 4, 1, 1) - - stats := ctrl.Wait() - - assert.NotEmpty(t, ctrl.PrintableStatus()) - // Expect 8 objects - assert.Equal(t, 8, stats.Objects) - // Expect 2 success - assert.Equal(t, 2, stats.Successes) - // Expect 2 folders - assert.Equal(t, 2, stats.Folders) -} - -func (suite *DisconnectedSuite) TestVerifyBackupInputs_allServices() { - sites := []string{"abc.site.foo", "bar.site.baz"} - - tests := []struct { - name string - excludes func(t *testing.T) selectors.Selector - filters func(t *testing.T) selectors.Selector - includes func(t *testing.T) selectors.Selector - checkError assert.ErrorAssertionFunc - }{ - { - name: "Valid User", - checkError: assert.NoError, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = "elliotReid@someHospital.org" - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Filter(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = "elliotReid@someHospital.org" - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Include(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = "elliotReid@someHospital.org" - return sel.Selector - }, - }, - { - name: "Invalid User", - checkError: assert.NoError, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Filter(selTD.OneDriveBackupFolderScope(sel)) - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Include(selTD.OneDriveBackupFolderScope(sel)) - return sel.Selector - }, - }, - { - name: "valid sites", - checkError: assert.NoError, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) - sel.DiscreteOwner = "abc.site.foo" - sel.Exclude(sel.AllData()) - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) - sel.DiscreteOwner = "abc.site.foo" - sel.Filter(sel.AllData()) - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) - sel.DiscreteOwner = "abc.site.foo" - sel.Include(sel.AllData()) - return sel.Selector - }, - }, - { - name: "invalid sites", - checkError: assert.Error, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) - sel.Exclude(sel.AllData()) - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) - sel.Filter(sel.AllData()) - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) - sel.Include(sel.AllData()) - return sel.Selector - }, - }, - } - - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - err := verifyBackupInputs(test.excludes(t), sites) - test.checkError(t, err, clues.ToCore(err)) - err = verifyBackupInputs(test.filters(t), sites) - test.checkError(t, err, clues.ToCore(err)) - err = verifyBackupInputs(test.includes(t), sites) - test.checkError(t, err, clues.ToCore(err)) - }) - } -} diff --git a/src/internal/m365/graph_connector_helper_test.go b/src/internal/m365/helper_test.go similarity index 98% rename from src/internal/m365/graph_connector_helper_test.go rename to src/internal/m365/helper_test.go index 1fc1573c7..fe1de81f8 100644 --- a/src/internal/m365/graph_connector_helper_test.go +++ b/src/internal/m365/helper_test.go @@ -19,7 +19,9 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" + m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -102,15 +104,15 @@ func testElementsMatch[T any]( type restoreBackupInfo struct { name string service path.ServiceType - collections []ColInfo + collections []m365Stub.ColInfo resourceCat resource.Category } type restoreBackupInfoMultiVersion struct { service path.ServiceType - collectionsLatest []ColInfo - collectionsPrevious []ColInfo - resource resource.Category + collectionsLatest []m365Stub.ColInfo + collectionsPrevious []m365Stub.ColInfo + resourceCat resource.Category backupVersion int } @@ -686,7 +688,7 @@ func compareDriveItem( t *testing.T, expected map[string][]byte, item data.Stream, - config ConfigInfo, + config m365Stub.ConfigInfo, rootDir bool, ) bool { // Skip Drive permissions in the folder that used to be the root. We don't @@ -793,7 +795,7 @@ func compareDriveItem( return true } - var fileData testOneDriveData + var fileData odStub.FileData err = json.Unmarshal(buf, &fileData) if !assert.NoError(t, err, "unmarshalling file data for file", name, clues.ToCore(err)) { @@ -829,7 +831,7 @@ func compareItem( service path.ServiceType, category path.CategoryType, item data.Stream, - config ConfigInfo, + config m365Stub.ConfigInfo, rootDir bool, ) bool { if mt, ok := item.(data.StreamModTime); ok { @@ -923,7 +925,7 @@ func checkCollections( expectedItems int, expected map[string]map[string][]byte, got []data.BackupCollection, - config ConfigInfo, + config m365Stub.ConfigInfo, ) int { collectionsWithItems := []data.BackupCollection{} @@ -985,7 +987,7 @@ func checkCollections( checkHasCollections(t, expected, collectionsWithItems) // Return how many metadata files were skipped so we can account for it in the - // check on Controller status. + // check on controller status. return skipped } @@ -1152,11 +1154,11 @@ func getSelectorWith( } } -func loadController(ctx context.Context, t *testing.T, r resource.Category) *Controller { +func newController(ctx context.Context, t *testing.T, r resource.Category) *Controller { a := tester.NewM365Account(t) - connector, err := NewController(ctx, a, r) + controller, err := NewController(ctx, a, r) require.NoError(t, err, clues.ToCore(err)) - return connector + return controller } diff --git a/src/internal/m365/mock/collection.go b/src/internal/m365/mock/collection.go new file mode 100644 index 000000000..e1d61db58 --- /dev/null +++ b/src/internal/m365/mock/collection.go @@ -0,0 +1,24 @@ +package mock + +import ( + "context" + + "github.com/alcionai/corso/src/internal/data" +) + +type RestoreCollection struct { + data.Collection + AuxItems map[string]data.Stream +} + +func (rc RestoreCollection) FetchItemByName( + ctx context.Context, + name string, +) (data.Stream, error) { + res := rc.AuxItems[name] + if res == nil { + return nil, data.ErrNotFound + } + + return res, nil +} diff --git a/src/internal/m365/onedrive/drive.go b/src/internal/m365/onedrive/item_collector.go similarity index 100% rename from src/internal/m365/onedrive/drive.go rename to src/internal/m365/onedrive/item_collector.go diff --git a/src/internal/m365/onedrive/drive_test.go b/src/internal/m365/onedrive/item_collector_test.go similarity index 98% rename from src/internal/m365/onedrive/drive_test.go rename to src/internal/m365/onedrive/item_collector_test.go index 348722b78..65e9bf5fe 100644 --- a/src/internal/m365/onedrive/drive_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -28,13 +28,12 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) -// Unit tests -type OneDriveUnitSuite struct { +type ItemCollectorUnitSuite struct { tester.Suite } func TestOneDriveUnitSuite(t *testing.T) { - suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)}) + suite.Run(t, &ItemCollectorUnitSuite{Suite: tester.NewUnitSuite(t)}) } const ( @@ -51,7 +50,7 @@ func odErr(code string) *odataerrors.ODataError { return odErr } -func (suite *OneDriveUnitSuite) TestDrives() { +func (suite *ItemCollectorUnitSuite) TestDrives() { t := suite.T() ctx, flush := tester.NewContext(t) diff --git a/src/internal/m365/graph_connector_onedrive_test_helper.go b/src/internal/m365/onedrive/stub/stub.go similarity index 59% rename from src/internal/m365/graph_connector_onedrive_test_helper.go rename to src/internal/m365/onedrive/stub/stub.go index 77acc1b7d..44590d5a1 100644 --- a/src/internal/m365/graph_connector_onedrive_test_helper.go +++ b/src/internal/m365/onedrive/stub/stub.go @@ -1,4 +1,4 @@ -package m365 +package stub import ( "encoding/json" @@ -6,11 +6,10 @@ import ( "github.com/alcionai/clues" "github.com/google/uuid" - "golang.org/x/exp/maps" - "github.com/alcionai/corso/src/internal/data" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" ) @@ -61,59 +60,59 @@ type ItemData struct { Perms PermData } -type OnedriveColInfo struct { +type ColInfo struct { PathElements []string Perms PermData Files []ItemData Folders []ItemData } -type onedriveCollection struct { - service path.ServiceType +type collection struct { + Service path.ServiceType PathElements []string - items []ItemInfo - aux []ItemInfo - backupVersion int + Items []m365Stub.ItemInfo + Aux []m365Stub.ItemInfo + BackupVersion int } -func (c onedriveCollection) collection() ColInfo { +func (c collection) ColInfo() m365Stub.ColInfo { cat := path.FilesCategory - if c.service == path.SharePointService { + if c.Service == path.SharePointService { cat = path.LibrariesCategory } - return ColInfo{ + return m365Stub.ColInfo{ PathElements: c.PathElements, Category: cat, - Items: c.items, - AuxItems: c.aux, + Items: c.Items, + AuxItems: c.Aux, } } -func NewOneDriveCollection( +func NewCollection( service path.ServiceType, PathElements []string, backupVersion int, -) *onedriveCollection { - return &onedriveCollection{ - service: service, +) *collection { + return &collection{ + Service: service, PathElements: PathElements, - backupVersion: backupVersion, + BackupVersion: backupVersion, } } func DataForInfo( service path.ServiceType, - cols []OnedriveColInfo, + cols []ColInfo, backupVersion int, -) ([]ColInfo, error) { +) ([]m365Stub.ColInfo, error) { var ( - res []ColInfo + res []m365Stub.ColInfo err error ) for _, c := range cols { - onedriveCol := NewOneDriveCollection(service, c.PathElements, backupVersion) + onedriveCol := NewCollection(service, c.PathElements, backupVersion) for _, f := range c.Files { _, err = onedriveCol.withFile(f.Name, f.Data, f.Perms) @@ -134,18 +133,18 @@ func DataForInfo( return res, err } - res = append(res, onedriveCol.collection()) + res = append(res, onedriveCol.ColInfo()) } return res, nil } -func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermData) (*onedriveCollection, error) { - switch c.backupVersion { +func (c *collection) withFile(name string, fileData []byte, perm PermData) (*collection, error) { + switch c.BackupVersion { case 0: // Lookups will occur using the most recent version of things so we need // the embedded file name to match that. - item, err := onedriveItemWithData( + item, err := FileWithData( name, name+metadata.DataFileSuffix, fileData) @@ -153,12 +152,12 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat return c, err } - c.items = append(c.items, item) + c.Items = append(c.Items, item) // v1-5, early metadata design case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName: - items, err := onedriveItemWithData( + items, err := FileWithData( name+metadata.DataFileSuffix, name+metadata.DataFileSuffix, fileData) @@ -166,24 +165,24 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat return c, err } - c.items = append(c.items, items) + c.Items = append(c.Items, items) - md, err := onedriveMetadata( + md, err := ItemWithMetadata( "", name+metadata.MetaFileSuffix, name+metadata.MetaFileSuffix, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) if err != nil { return c, err } - c.items = append(c.items, md) - c.aux = append(c.aux, md) + c.Items = append(c.Items, md) + c.Aux = append(c.Aux, md) // v6+ current metadata design case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID: - item, err := onedriveItemWithData( + item, err := FileWithData( name+metadata.DataFileSuffix, name+metadata.DataFileSuffix, fileData) @@ -191,50 +190,50 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat return c, err } - c.items = append(c.items, item) + c.Items = append(c.Items, item) - md, err := onedriveMetadata( + md, err := ItemWithMetadata( name, name+metadata.MetaFileSuffix, name, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) if err != nil { return c, err } - c.items = append(c.items, md) - c.aux = append(c.aux, md) + c.Items = append(c.Items, md) + c.Aux = append(c.Aux, md) default: - return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.backupVersion)) + return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.BackupVersion)) } return c, nil } -func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCollection, error) { - switch c.backupVersion { +func (c *collection) withFolder(name string, perm PermData) (*collection, error) { + switch c.BackupVersion { case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName, version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID: return c, nil case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker: - item, err := onedriveMetadata( + item, err := ItemWithMetadata( "", name+metadata.DirMetaFileSuffix, name+metadata.DirMetaFileSuffix, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, item) + c.Items = append(c.Items, item) if err != nil { return c, err } default: - return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.backupVersion)) + return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.BackupVersion)) } return c, nil @@ -242,17 +241,17 @@ func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCo // withPermissions adds permissions to the folder represented by this // onedriveCollection. -func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection, error) { +func (c *collection) withPermissions(perm PermData) (*collection, error) { // These versions didn't store permissions for the folder or didn't store them // in the folder's collection. - if c.backupVersion < version.OneDrive4DirIncludesPermissions { + if c.BackupVersion < version.OneDrive4DirIncludesPermissions { return c, nil } name := c.PathElements[len(c.PathElements)-1] metaName := name - if c.backupVersion >= version.OneDrive5DirMetaNoName { + if c.BackupVersion >= version.OneDrive5DirMetaNoName { // We switched to just .dirmeta for metadata file names. metaName = "" } @@ -261,98 +260,63 @@ func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection return c, nil } - md, err := onedriveMetadata( + md, err := ItemWithMetadata( name, metaName+metadata.DirMetaFileSuffix, metaName+metadata.DirMetaFileSuffix, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) if err != nil { return c, err } - c.items = append(c.items, md) - c.aux = append(c.aux, md) + c.Items = append(c.Items, md) + c.Aux = append(c.Aux, md) return c, err } -type testOneDriveData struct { +type FileData struct { FileName string `json:"fileName,omitempty"` Data []byte `json:"data,omitempty"` } -func onedriveItemWithData( +func FileWithData( name, lookupKey string, fileData []byte, -) (ItemInfo, error) { - content := testOneDriveData{ +) (m365Stub.ItemInfo, error) { + content := FileData{ FileName: lookupKey, Data: fileData, } serialized, err := json.Marshal(content) if err != nil { - return ItemInfo{}, clues.Stack(err) + return m365Stub.ItemInfo{}, clues.Stack(err) } - return ItemInfo{ - name: name, - data: serialized, - lookupKey: lookupKey, + return m365Stub.ItemInfo{ + Name: name, + Data: serialized, + LookupKey: lookupKey, }, nil } -func onedriveMetadata( +func ItemWithMetadata( fileName, itemID, lookupKey string, perm PermData, permUseID bool, -) (ItemInfo, error) { +) (m365Stub.ItemInfo, error) { testMeta := getMetadata(fileName, perm, permUseID) testMetaJSON, err := json.Marshal(testMeta) if err != nil { - return ItemInfo{}, clues.Wrap(err, "marshalling metadata") + return m365Stub.ItemInfo{}, clues.Wrap(err, "marshalling metadata") } - return ItemInfo{ - name: itemID, - data: testMetaJSON, - lookupKey: lookupKey, + return m365Stub.ItemInfo{ + Name: itemID, + Data: testMetaJSON, + LookupKey: lookupKey, }, nil } - -func GetCollectionsAndExpected( - config ConfigInfo, - testCollections []ColInfo, - backupVersion int, -) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { - var ( - collections []data.RestoreCollection - expectedData = map[string]map[string][]byte{} - totalItems = 0 - totalKopiaItems = 0 - ) - - for _, owner := range config.ResourceOwners { - numItems, kopiaItems, ownerCollections, userExpectedData, err := collectionsForInfo( - config.Service, - config.Tenant, - owner, - config.RestoreCfg, - testCollections, - backupVersion, - ) - if err != nil { - return totalItems, totalKopiaItems, collections, expectedData, err - } - - collections = append(collections, ownerCollections...) - totalItems += numItems - totalKopiaItems += kopiaItems - - maps.Copy(expectedData, userExpectedData) - } - - return totalItems, totalKopiaItems, collections, expectedData, nil -} diff --git a/src/internal/m365/graph_connector_onedrive_test.go b/src/internal/m365/onedrive_test.go similarity index 88% rename from src/internal/m365/graph_connector_onedrive_test.go rename to src/internal/m365/onedrive_test.go index 1a45992d3..ca821392f 100644 --- a/src/internal/m365/graph_connector_onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -96,17 +97,17 @@ type oneDriveSuite interface { } type suiteInfoImpl struct { - ac api.Client - controller *Controller - resourceOwner string - resourceCat resource.Category - secondaryUser string - secondaryUserID string - service path.ServiceType - tertiaryUser string - tertiaryUserID string - user string - userID string + ac api.Client + controller *Controller + resourceOwner string + resourceCategory resource.Category + secondaryUser string + secondaryUserID string + service path.ServiceType + tertiaryUser string + tertiaryUserID string + user string + userID string } func NewSuiteInfoImpl( @@ -115,22 +116,22 @@ func NewSuiteInfoImpl( resourceOwner string, service path.ServiceType, ) suiteInfoImpl { - rc := resource.Users + rsc := resource.Users if service == path.SharePointService { - rc = resource.Sites + rsc = resource.Sites } - ctrl := loadController(ctx, t, rc) + ctrl := newController(ctx, t, rsc) return suiteInfoImpl{ - ac: ctrl.AC, - controller: ctrl, - resourceOwner: resourceOwner, - resourceCat: rc, - secondaryUser: tester.SecondaryM365UserID(t), - service: service, - tertiaryUser: tester.TertiaryM365UserID(t), - user: tester.M365UserID(t), + ac: ctrl.AC, + controller: ctrl, + resourceOwner: resourceOwner, + resourceCategory: rsc, + secondaryUser: tester.SecondaryM365UserID(t), + service: service, + tertiaryUser: tester.TertiaryM365UserID(t), + user: tester.M365UserID(t), } } @@ -163,7 +164,7 @@ func (si suiteInfoImpl) Service() path.ServiceType { } func (si suiteInfoImpl) Resource() resource.Category { - return si.resourceCat + return si.resourceCategory } // --------------------------------------------------------------------------- @@ -402,16 +403,16 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( folderBName, } - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderAName, }, @@ -422,13 +423,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: folderAPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileBData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderBName, }, @@ -436,13 +437,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: subfolderBPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileCData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderAName, }, @@ -450,7 +451,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: subfolderAPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileDData, @@ -459,7 +460,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: folderBPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileEData, @@ -468,18 +469,18 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) for vn := startVersion; vn <= version.Backup; vn++ { suite.Run(fmt.Sprintf("Version%d", vn), func() { t := suite.T() - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -549,15 +550,15 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { folderCName, } - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{ + Files: []stub.ItemData{ { // Test restoring a file that doesn't inherit permissions. Name: fileName, Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -570,13 +571,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { Data: fileBData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderBName, }, { Name: folderAName, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -584,7 +585,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { }, { Name: folderCName, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -594,23 +595,23 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { }, { PathElements: folderBPath, - Files: []ItemData{ + Files: []stub.ItemData{ { // Test restoring a file in a non-root folder that doesn't inherit // permissions. Name: fileName, Data: fileBData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderAName, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -624,18 +625,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // // Tests a folder that has permissions with an item in the folder with // // the same permissions. // pathElements: subfolderAPath, - // files: []itemData{ + // files: []stub.ItemData{ // { // name: fileName, // data: fileDData, - // perms: permData{ + // perms: stub.PermData{ // user: secondaryUserName, // entityID: secondaryUserID, // roles: readPerm, // }, // }, // }, - // Perms: PermData{ + // Perms: stub.PermData{ // User: secondaryUserName, // EntityID: secondaryUserID, // Roles: readPerm, @@ -645,18 +646,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // Tests a folder that has permissions with an item in the folder with // the different permissions. PathElements: folderAPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileEData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -666,13 +667,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // Tests a folder that has permissions with an item in the folder with // no permissions. PathElements: folderCPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, }, }, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -680,7 +681,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() @@ -690,12 +691,12 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // Ideally this can always be true or false and still // work, but limiting older versions to use emails so as // to validate that flow as well. - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -730,18 +731,18 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { suite.Service(), suite.ResourceOwner()) - inputCols := []OnedriveColInfo{ + inputCols := []stub.ColInfo{ { PathElements: []string{ odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, }, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -751,14 +752,14 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { }, } - expectedCols := []OnedriveColInfo{ + expectedCols := []stub.ColInfo{ { PathElements: []string{ odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, }, - Files: []ItemData{ + Files: []stub.ItemData{ { // No permissions on the output since they weren't restored. Name: fileName, @@ -768,19 +769,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { }, } - expected, err := DataForInfo(suite.Service(), expectedCols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), expectedCols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() for vn := startVersion; vn <= version.Backup; vn++ { suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() { t := suite.T() - input, err := DataForInfo(suite.Service(), inputCols, vn) + input, err := stub.DataForInfo(suite.Service(), inputCols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -855,11 +856,11 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio folderCName, } - fileSet := []ItemData{ + fileSet := []stub.ItemData{ { Name: "file-custom", Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -869,14 +870,14 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio { Name: "file-inherited", Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeInherited, }, }, { Name: "file-empty", Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeCustom, }, }, @@ -900,23 +901,23 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio // - inherted-permission-file // - empty-permission-file (empty/empty might have interesting behavior) - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{}, - Folders: []ItemData{ + Files: []stub.ItemData{}, + Folders: []stub.ItemData{ {Name: folderAName}, }, }, { PathElements: folderAPath, Files: fileSet, - Folders: []ItemData{ + Folders: []stub.ItemData{ {Name: folderAName}, {Name: folderBName}, {Name: folderCName}, }, - Perms: PermData{ + Perms: stub.PermData{ User: tertiaryUserName, EntityID: tertiaryUserID, Roles: readPerm, @@ -925,7 +926,7 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio { PathElements: subfolderAAPath, Files: fileSet, - Perms: PermData{ + Perms: stub.PermData{ User: tertiaryUserName, EntityID: tertiaryUserID, Roles: writePerm, @@ -935,20 +936,20 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio { PathElements: subfolderABPath, Files: fileSet, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeInherited, }, }, { PathElements: subfolderACPath, Files: fileSet, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeCustom, }, }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() @@ -958,12 +959,12 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio // Ideally this can always be true or false and still // work, but limiting older versions to use emails so as // to validate that flow as well. - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -1018,16 +1019,16 @@ func testRestoreFolderNamedFolderRegression( folderBName, } - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderNamedFolder, }, @@ -1038,13 +1039,13 @@ func testRestoreFolderNamedFolderRegression( }, { PathElements: folderFolderPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileBData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderBName, }, @@ -1052,13 +1053,13 @@ func testRestoreFolderNamedFolderRegression( }, { PathElements: subfolderPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileCData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderNamedFolder, }, @@ -1066,19 +1067,19 @@ func testRestoreFolderNamedFolderRegression( }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() for vn := startVersion; vn <= version.Backup; vn++ { suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() { t := suite.T() - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, diff --git a/src/internal/m365/sharepoint/api/pages.go b/src/internal/m365/sharepoint/api/pages.go index 581d5de10..e7d24fd6f 100644 --- a/src/internal/m365/sharepoint/api/pages.go +++ b/src/internal/m365/sharepoint/api/pages.go @@ -172,7 +172,7 @@ func RestoreSitePage( itemData data.Stream, siteID, destName string, ) (details.ItemInfo, error) { - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID())) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID())) defer end() var ( diff --git a/src/internal/m365/sharepoint/collection.go b/src/internal/m365/sharepoint/collection.go index 90af58cbf..687a2ebb8 100644 --- a/src/internal/m365/sharepoint/collection.go +++ b/src/internal/m365/sharepoint/collection.go @@ -257,7 +257,7 @@ func (sc *Collection) retrieveLists( sc.data <- &Item{ id: ptr.Val(lst.GetId()), data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointListInfo(lst, size), + info: listToSPInfo(lst, size), modTime: t, } @@ -320,7 +320,7 @@ func (sc *Collection) retrievePages( sc.data <- &Item{ id: ptr.Val(pg.GetId()), data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointPageInfo(pg, root, size), + info: pageToSPInfo(pg, root, size), modTime: ptr.OrNow(pg.GetLastModifiedDateTime()), } diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/sharepoint/collection_test.go index 5381710c2..5177ecf06 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/sharepoint/collection_test.go @@ -116,7 +116,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { data := &Item{ id: name, data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointListInfo(listing, int64(len(byteArray))), + info: listToSPInfo(listing, int64(len(byteArray))), } return data @@ -205,7 +205,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { listData := &Item{ id: testName, data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointListInfo(listing, int64(len(byteArray))), + info: listToSPInfo(listing, int64(len(byteArray))), } destName := tester.DefaultTestRestoreConfig("").Location diff --git a/src/internal/m365/sharepoint/list_info.go b/src/internal/m365/sharepoint/list_info.go deleted file mode 100644 index 62101b584..000000000 --- a/src/internal/m365/sharepoint/list_info.go +++ /dev/null @@ -1,28 +0,0 @@ -package sharepoint - -import ( - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/pkg/backup/details" -) - -// sharePointListInfo translates models.Listable metadata into searchable content -// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 -func sharePointListInfo(lst models.Listable, size int64) *details.SharePointInfo { - var ( - name = ptr.Val(lst.GetDisplayName()) - webURL = ptr.Val(lst.GetWebUrl()) - created = ptr.Val(lst.GetCreatedDateTime()) - modified = ptr.Val(lst.GetLastModifiedDateTime()) - ) - - return &details.SharePointInfo{ - ItemType: details.SharePointList, - ItemName: name, - Created: created, - Modified: modified, - WebURL: webURL, - Size: size, - } -} diff --git a/src/internal/m365/sharepoint/list_info_test.go b/src/internal/m365/sharepoint/list_info_test.go deleted file mode 100644 index a4467450f..000000000 --- a/src/internal/m365/sharepoint/list_info_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package sharepoint - -import ( - "testing" - - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/backup/details" -) - -type SharePointInfoSuite struct { - tester.Suite -} - -func TestSharePointInfoSuite(t *testing.T) { - suite.Run(t, &SharePointInfoSuite{Suite: tester.NewUnitSuite(t)}) -} - -func (suite *SharePointInfoSuite) TestSharePointInfo() { - tests := []struct { - name string - listAndDeets func() (models.Listable, *details.SharePointInfo) - }{ - { - name: "Empty List", - listAndDeets: func() (models.Listable, *details.SharePointInfo) { - i := &details.SharePointInfo{ItemType: details.SharePointList} - return models.NewList(), i - }, - }, { - name: "Only Name", - listAndDeets: func() (models.Listable, *details.SharePointInfo) { - aTitle := "Whole List" - listing := models.NewList() - listing.SetDisplayName(&aTitle) - i := &details.SharePointInfo{ - ItemType: details.SharePointList, - ItemName: aTitle, - } - - return listing, i - }, - }, - } - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - list, expected := test.listAndDeets() - info := sharePointListInfo(list, 10) - assert.Equal(t, expected.ItemType, info.ItemType) - assert.Equal(t, expected.ItemName, info.ItemName) - assert.Equal(t, expected.WebURL, info.WebURL) - }) - } -} diff --git a/src/internal/m365/sharepoint/list.go b/src/internal/m365/sharepoint/lists.go similarity index 93% rename from src/internal/m365/sharepoint/list.go rename to src/internal/m365/sharepoint/lists.go index 3dcaaa58f..0555516af 100644 --- a/src/internal/m365/sharepoint/list.go +++ b/src/internal/m365/sharepoint/lists.go @@ -10,9 +10,30 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" ) +// listToSPInfo translates models.Listable metadata into searchable content +// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 +func listToSPInfo(lst models.Listable, size int64) *details.SharePointInfo { + var ( + name = ptr.Val(lst.GetDisplayName()) + webURL = ptr.Val(lst.GetWebUrl()) + created = ptr.Val(lst.GetCreatedDateTime()) + modified = ptr.Val(lst.GetLastModifiedDateTime()) + ) + + return &details.SharePointInfo{ + ItemType: details.SharePointList, + ItemName: name, + Created: created, + Modified: modified, + WebURL: webURL, + Size: size, + } +} + type listTuple struct { name string id string diff --git a/src/internal/m365/sharepoint/list_test.go b/src/internal/m365/sharepoint/lists_test.go similarity index 53% rename from src/internal/m365/sharepoint/list_test.go rename to src/internal/m365/sharepoint/lists_test.go index 05990712d..32cb16918 100644 --- a/src/internal/m365/sharepoint/list_test.go +++ b/src/internal/m365/sharepoint/lists_test.go @@ -4,21 +4,23 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" ) -type SharePointSuite struct { +type ListsUnitSuite struct { tester.Suite creds account.M365Config } -func (suite *SharePointSuite) SetupSuite() { +func (suite *ListsUnitSuite) SetupSuite() { t := suite.T() a := tester.NewM365Account(t) m365, err := a.M365Config() @@ -27,8 +29,8 @@ func (suite *SharePointSuite) SetupSuite() { suite.creds = m365 } -func TestSharePointSuite(t *testing.T) { - suite.Run(t, &SharePointSuite{ +func TestListsUnitSuite(t *testing.T) { + suite.Run(t, &ListsUnitSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tester.M365AcctCredEnvs}, @@ -47,7 +49,7 @@ func TestSharePointSuite(t *testing.T) { // to verify if these 2 calls are valid // - fetchContentBaseTypes // - fetchColumnPositions -func (suite *SharePointSuite) TestLoadList() { +func (suite *ListsUnitSuite) TestLoadList() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -63,3 +65,42 @@ func (suite *SharePointSuite) TestLoadList() { assert.Greater(t, len(lists), 0) t.Logf("Length: %d\n", len(lists)) } + +func (suite *ListsUnitSuite) TestSharePointInfo() { + tests := []struct { + name string + listAndDeets func() (models.Listable, *details.SharePointInfo) + }{ + { + name: "Empty List", + listAndDeets: func() (models.Listable, *details.SharePointInfo) { + i := &details.SharePointInfo{ItemType: details.SharePointList} + return models.NewList(), i + }, + }, { + name: "Only Name", + listAndDeets: func() (models.Listable, *details.SharePointInfo) { + aTitle := "Whole List" + listing := models.NewList() + listing.SetDisplayName(&aTitle) + i := &details.SharePointInfo{ + ItemType: details.SharePointList, + ItemName: aTitle, + } + + return listing, i + }, + }, + } + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + list, expected := test.listAndDeets() + info := listToSPInfo(list, 10) + assert.Equal(t, expected.ItemType, info.ItemType) + assert.Equal(t, expected.ItemName, info.ItemName) + assert.Equal(t, expected.WebURL, info.WebURL) + }) + } +} diff --git a/src/internal/m365/sharepoint/pageInfo.go b/src/internal/m365/sharepoint/pages.go similarity index 84% rename from src/internal/m365/sharepoint/pageInfo.go rename to src/internal/m365/sharepoint/pages.go index 8b5060bdd..c5e0bb633 100644 --- a/src/internal/m365/sharepoint/pageInfo.go +++ b/src/internal/m365/sharepoint/pages.go @@ -8,10 +8,10 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" ) -// sharePointPageInfo propagates metadata from the SharePoint Page data type +// pageToSPInfo propagates metadata from the SharePoint Page data type // into searchable content. // Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta -func sharePointPageInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo { +func pageToSPInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo { var ( name, prefix, webURL string created, modified time.Time diff --git a/src/internal/m365/sharepoint/pageInfo_test.go b/src/internal/m365/sharepoint/pages_test.go similarity index 76% rename from src/internal/m365/sharepoint/pageInfo_test.go rename to src/internal/m365/sharepoint/pages_test.go index 7490b117f..d89b0d921 100644 --- a/src/internal/m365/sharepoint/pageInfo_test.go +++ b/src/internal/m365/sharepoint/pages_test.go @@ -1,13 +1,25 @@ package sharepoint import ( + "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/m365/graph/betasdk/models" + "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" ) -func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { +type PagesUnitSuite struct { + tester.Suite +} + +func TestPagesUnitSuite(t *testing.T) { + suite.Run(t, &PagesUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *PagesUnitSuite) TestSharePointInfo_Pages() { tests := []struct { name string pageAndDeets func() (models.SitePageable, *details.SharePointInfo) @@ -39,7 +51,7 @@ func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { t := suite.T() paged, expected := test.pageAndDeets() - info := sharePointPageInfo(paged, "", 0) + info := pageToSPInfo(paged, "", 0) assert.Equal(t, expected.ItemType, info.ItemType) assert.Equal(t, expected.ItemName, info.ItemName) assert.Equal(t, expected.WebURL, info.WebURL) diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index 7076d4de9..772515fd4 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -129,7 +129,7 @@ func restoreListItem( itemData data.Stream, siteID, destName string, ) (details.ItemInfo, error) { - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID())) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID())) defer end() ctx = clues.Add(ctx, "list_item_id", itemData.UUID()) @@ -190,7 +190,7 @@ func restoreListItem( } } - dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray))) + dii.SharePoint = listToSPInfo(restoredList, int64(len(byteArray))) return dii, nil } @@ -203,7 +203,7 @@ func RestoreListCollection( deets *details.Builder, errs *fault.Bus, ) (support.CollectionMetrics, error) { - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath())) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath())) defer end() var ( @@ -214,7 +214,7 @@ func RestoreListCollection( el = errs.Local() ) - trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String()) + trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String()) for { if el.Failure() != nil { @@ -285,8 +285,8 @@ func RestorePageCollection( siteID = directory.ResourceOwner() ) - trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String()) - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath())) + trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String()) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath())) defer end() diff --git a/src/internal/m365/graph_connector_test_helper.go b/src/internal/m365/stub/stub.go similarity index 71% rename from src/internal/m365/graph_connector_test_helper.go rename to src/internal/m365/stub/stub.go index f95d1781f..601e57722 100644 --- a/src/internal/m365/graph_connector_test_helper.go +++ b/src/internal/m365/stub/stub.go @@ -1,12 +1,14 @@ -package m365 +package stub import ( "bytes" - "context" "io" + "golang.org/x/exp/maps" + "github.com/alcionai/corso/src/internal/data" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/pkg/control" @@ -33,9 +35,9 @@ type ItemInfo struct { // be the same before and after restoring the item in M365 and may not be // the M365 ID. When restoring items out of place, the item is assigned a // new ID making it unsuitable for a lookup key. - lookupKey string - name string - data []byte + LookupKey string + Name string + Data []byte } type ConfigInfo struct { @@ -47,19 +49,113 @@ type ConfigInfo struct { RestoreCfg control.RestoreConfig } -func mustToDataLayerPath( - service path.ServiceType, - tenant, resourceOwner string, - category path.CategoryType, - elements []string, - isItem bool, -) (path.Path, error) { - res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...) - if err != nil { - return nil, err +func GetCollectionsAndExpected( + config ConfigInfo, + testCollections []ColInfo, + backupVersion int, +) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { + var ( + collections []data.RestoreCollection + expectedData = map[string]map[string][]byte{} + totalItems = 0 + totalKopiaItems = 0 + ) + + for _, owner := range config.ResourceOwners { + numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo( + config.Service, + config.Tenant, + owner, + config.RestoreCfg, + testCollections, + backupVersion, + ) + if err != nil { + return totalItems, totalKopiaItems, collections, expectedData, err + } + + collections = append(collections, ownerCollections...) + totalItems += numItems + totalKopiaItems += kopiaItems + + maps.Copy(expectedData, userExpectedData) } - return res, err + return totalItems, totalKopiaItems, collections, expectedData, nil +} + +func CollectionsForInfo( + service path.ServiceType, + tenant, user string, + restoreCfg control.RestoreConfig, + allInfo []ColInfo, + backupVersion int, +) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { + var ( + collections = make([]data.RestoreCollection, 0, len(allInfo)) + expectedData = make(map[string]map[string][]byte, len(allInfo)) + totalItems = 0 + kopiaEntries = 0 + ) + + for _, info := range allInfo { + pth, err := path.Build( + tenant, + user, + service, + info.Category, + false, + info.PathElements...) + if err != nil { + return totalItems, kopiaEntries, collections, expectedData, err + } + + mc := exchMock.NewCollection(pth, pth, len(info.Items)) + + baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth) + if err != nil { + return totalItems, kopiaEntries, collections, expectedData, err + } + + baseExpected := expectedData[baseDestPath.String()] + if baseExpected == nil { + expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items)) + baseExpected = expectedData[baseDestPath.String()] + } + + for i := 0; i < len(info.Items); i++ { + mc.Names[i] = info.Items[i].Name + mc.Data[i] = info.Items[i].Data + + baseExpected[info.Items[i].LookupKey] = info.Items[i].Data + + // We do not count metadata files against item count + if backupVersion > 0 && + (service == path.OneDriveService || service == path.SharePointService) && + metadata.HasMetaSuffix(info.Items[i].Name) { + continue + } + + totalItems++ + } + + c := mock.RestoreCollection{ + Collection: mc, + AuxItems: map[string]data.Stream{}, + } + + for _, aux := range info.AuxItems { + c.AuxItems[aux.Name] = &exchMock.Data{ + ID: aux.Name, + Reader: io.NopCloser(bytes.NewReader(aux.Data)), + } + } + + collections = append(collections, c) + kopiaEntries += len(info.Items) + } + + return totalItems, kopiaEntries, collections, expectedData, nil } // backupOutputPathFromRestore returns a path.Path denoting the location in @@ -86,102 +182,11 @@ func backupOutputPathFromRestore( base = append(base, inputPath.Folders()...) } - return mustToDataLayerPath( - inputPath.Service(), + return path.Build( inputPath.Tenant(), inputPath.ResourceOwner(), + inputPath.Service(), inputPath.Category(), - base, false, - ) -} - -// TODO(ashmrtn): Make this an actual mock class that can be used in other -// packages. -type mockRestoreCollection struct { - data.Collection - auxItems map[string]data.Stream -} - -func (rc mockRestoreCollection) FetchItemByName( - ctx context.Context, - name string, -) (data.Stream, error) { - res := rc.auxItems[name] - if res == nil { - return nil, data.ErrNotFound - } - - return res, nil -} - -func collectionsForInfo( - service path.ServiceType, - tenant, user string, - restoreCfg control.RestoreConfig, - allInfo []ColInfo, - backupVersion int, -) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { - var ( - collections = make([]data.RestoreCollection, 0, len(allInfo)) - expectedData = make(map[string]map[string][]byte, len(allInfo)) - totalItems = 0 - kopiaEntries = 0 - ) - - for _, info := range allInfo { - pth, err := mustToDataLayerPath( - service, - tenant, - user, - info.Category, - info.PathElements, - false) - if err != nil { - return totalItems, kopiaEntries, collections, expectedData, err - } - - mc := exchMock.NewCollection(pth, pth, len(info.Items)) - - baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth) - if err != nil { - return totalItems, kopiaEntries, collections, expectedData, err - } - - baseExpected := expectedData[baseDestPath.String()] - if baseExpected == nil { - expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items)) - baseExpected = expectedData[baseDestPath.String()] - } - - for i := 0; i < len(info.Items); i++ { - mc.Names[i] = info.Items[i].name - mc.Data[i] = info.Items[i].data - - baseExpected[info.Items[i].lookupKey] = info.Items[i].data - - // We do not count metadata files against item count - if backupVersion > 0 && - (service == path.OneDriveService || service == path.SharePointService) && - metadata.HasMetaSuffix(info.Items[i].name) { - continue - } - - totalItems++ - } - - c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}} - - for _, aux := range info.AuxItems { - c.auxItems[aux.name] = &exchMock.Data{ - ID: aux.name, - Reader: io.NopCloser(bytes.NewReader(aux.data)), - } - } - - collections = append(collections, c) - kopiaEntries += len(info.Items) - } - - return totalItems, kopiaEntries, collections, expectedData, nil + base...) }