From bd0f6f9769b6f7c0d94cfec59e8064f8b14a754c Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 14 Jun 2023 10:50:17 -0600 Subject: [PATCH] final cleanup for renaming/movement (#3612) This last step in the cleanup has two goals: first to minimize the number of arbitrarily named and located files so that code is better condensed and co-located. This is all just file renaming and a minor amount of code copy-pasting. The second is to create a centralized package to own the ColInfo type structs that we use to both stub out test data, and also generate factory data for cmds. The current ownership is haphazard, and while this movement is a little more condensed, it's still jumping through some weird hoops to get things to work. Treat it as one good step forward, and we'll have to return to polish it another time. At least it'll be separated from the m365 folder at large this way, and more easily identified as supporting design rather than production usage. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Issue(s) * closes #1996 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/factory/impl/common.go | 64 ++-- src/internal/m365/backup_test.go | 12 +- src/internal/m365/controller_test.go | 349 +++++++++++++----- ...r_cache.go => contacts_container_cache.go} | 0 ...ner_cache.go => events_container_cache.go} | 0 .../m365/graph_connector_disconnected_test.go | 181 --------- ...onnector_helper_test.go => helper_test.go} | 26 +- src/internal/m365/mock/collection.go | 24 ++ .../onedrive/{drive.go => item_collector.go} | 0 .../{drive_test.go => item_collector_test.go} | 7 +- .../stub/stub.go} | 174 ++++----- ...ctor_onedrive_test.go => onedrive_test.go} | 179 ++++----- src/internal/m365/sharepoint/api/pages.go | 2 +- src/internal/m365/sharepoint/collection.go | 4 +- .../m365/sharepoint/collection_test.go | 4 +- src/internal/m365/sharepoint/list_info.go | 28 -- .../m365/sharepoint/list_info_test.go | 59 --- .../m365/sharepoint/{list.go => lists.go} | 21 ++ .../{list_test.go => lists_test.go} | 51 ++- .../m365/sharepoint/{pageInfo.go => pages.go} | 4 +- .../{pageInfo_test.go => pages_test.go} | 16 +- src/internal/m365/sharepoint/restore.go | 12 +- .../stub.go} | 225 +++++------ 23 files changed, 705 insertions(+), 737 deletions(-) rename src/internal/m365/exchange/{contact_container_cache.go => contacts_container_cache.go} (100%) rename src/internal/m365/exchange/{event_container_cache.go => events_container_cache.go} (100%) delete mode 100644 src/internal/m365/graph_connector_disconnected_test.go rename src/internal/m365/{graph_connector_helper_test.go => helper_test.go} (98%) create mode 100644 src/internal/m365/mock/collection.go rename src/internal/m365/onedrive/{drive.go => item_collector.go} (100%) rename src/internal/m365/onedrive/{drive_test.go => item_collector_test.go} (98%) rename src/internal/m365/{graph_connector_onedrive_test_helper.go => onedrive/stub/stub.go} (59%) rename src/internal/m365/{graph_connector_onedrive_test.go => onedrive_test.go} (88%) delete mode 100644 src/internal/m365/sharepoint/list_info.go delete mode 100644 src/internal/m365/sharepoint/list_info_test.go rename src/internal/m365/sharepoint/{list.go => lists.go} (93%) rename src/internal/m365/sharepoint/{list_test.go => lists_test.go} (53%) rename src/internal/m365/sharepoint/{pageInfo.go => pages.go} (84%) rename src/internal/m365/sharepoint/{pageInfo_test.go => pages_test.go} (76%) rename src/internal/m365/{graph_connector_test_helper.go => stub/stub.go} (71%) diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index f1b863275..d1855b108 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -18,7 +18,9 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" + m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" @@ -84,14 +86,14 @@ func generateAndRestoreItems( items: items, }} - dest := control.DefaultRestoreConfig(dttm.SafeForTesting) - dest.Location = destFldr - print.Infof(ctx, "Restoring to folder %s", dest.Location) + restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) + restoreCfg.Location = destFldr + print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location) dataColls, err := buildCollections( service, tenantID, userID, - dest, + restoreCfg, collections) if err != nil { return nil, err @@ -99,7 +101,7 @@ func generateAndRestoreItems( print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, dataColls, errs) + return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs) } // ------------------------------------------------------------------------------------------ @@ -108,7 +110,7 @@ func generateAndRestoreItems( func getControllerAndVerifyResourceOwner( ctx context.Context, - rc resource.Category, + resourceCat resource.Category, resourceOwner string, ) ( *m365.Controller, @@ -133,7 +135,7 @@ func getControllerAndVerifyResourceOwner( return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details") } - ctrl, err := m365.NewController(ctx, acct, rc) + ctrl, err := m365.NewController(ctx, acct, resourceCat) if err != nil { return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") } @@ -164,7 +166,7 @@ type collection struct { func buildCollections( service path.ServiceType, tenant, user string, - dest control.RestoreConfig, + restoreCfg control.RestoreConfig, colls []collection, ) ([]data.RestoreCollection, error) { collections := make([]data.RestoreCollection, 0, len(colls)) @@ -225,9 +227,9 @@ func generateAndRestoreDriveItems( ctx, flush := tester.NewContext(nil) defer flush() - dest := control.DefaultRestoreConfig(dttm.SafeForTesting) - dest.Location = destFldr - print.Infof(ctx, "Restoring to folder %s", dest.Location) + restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) + restoreCfg.Location = destFldr + print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location) var driveID string @@ -249,7 +251,7 @@ func generateAndRestoreDriveItems( } var ( - cols []m365.OnedriveColInfo + cols []odStub.ColInfo rootPath = []string{"drives", driveID, "root:"} folderAPath = []string{"drives", driveID, "root:", folderAName} @@ -263,15 +265,15 @@ func generateAndRestoreDriveItems( ) for i := 0; i < count; i++ { - col := []m365.OnedriveColInfo{ + col := []odStub.ColInfo{ // basic folder and file creation { PathElements: rootPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime), Data: fileAData, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -282,13 +284,13 @@ func generateAndRestoreDriveItems( Data: fileBData, }, }, - Folders: []m365.ItemData{ + Folders: []odStub.ItemData{ { Name: folderBName, }, { Name: folderAName, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -296,7 +298,7 @@ func generateAndRestoreDriveItems( }, { Name: folderCName, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -308,18 +310,18 @@ func generateAndRestoreDriveItems( // a folder that has permissions with an item in the folder with // the different permissions. PathElements: folderAPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Data: fileEData, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -329,13 +331,13 @@ func generateAndRestoreDriveItems( // a folder that has permissions with an item in the folder with // no permissions. PathElements: folderCPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Data: fileAData, }, }, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -343,23 +345,23 @@ func generateAndRestoreDriveItems( }, { PathElements: folderBPath, - Files: []m365.ItemData{ + Files: []odStub.ItemData{ { // restoring a file in a non-root folder that doesn't inherit // permissions. Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Data: fileBData, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Folders: []m365.ItemData{ + Folders: []odStub.ItemData{ { Name: folderAName, - Perms: m365.PermData{ + Perms: odStub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -372,7 +374,7 @@ func generateAndRestoreDriveItems( cols = append(cols, col...) } - input, err := m365.DataForInfo(service, cols, version.Backup) + input, err := odStub.DataForInfo(service, cols, version.Backup) if err != nil { return nil, err } @@ -389,7 +391,7 @@ func generateAndRestoreDriveItems( ToggleFeatures: control.Toggles{}, } - config := m365.ConfigInfo{ + config := m365Stub.ConfigInfo{ Opts: opts, Resource: resource.Users, Service: service, @@ -398,7 +400,7 @@ func generateAndRestoreDriveItems( RestoreCfg: tester.DefaultTestRestoreConfig(""), } - _, _, collections, _, err := m365.GetCollectionsAndExpected( + _, _, collections, _, err := m365Stub.GetCollectionsAndExpected( config, input, version.Backup) @@ -406,5 +408,5 @@ func generateAndRestoreDriveItems( return nil, err } - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, collections, errs) + return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs) } diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 9429be012..03543061b 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -66,7 +66,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { selUsers := []string{suite.user} - ctrl := loadController(ctx, suite.T(), resource.Users) + ctrl := newController(ctx, suite.T(), resource.Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -167,7 +167,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() defer flush() owners := []string{"snuffleupagus"} - ctrl := loadController(ctx, suite.T(), resource.Users) + ctrl := newController(ctx, suite.T(), resource.Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -253,7 +253,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { defer flush() selSites := []string{suite.site} - ctrl := loadController(ctx, suite.T(), resource.Sites) + ctrl := newController(ctx, suite.T(), resource.Sites) tests := []struct { name string expected int @@ -348,7 +348,7 @@ func (suite *SPCollectionIntgSuite) SetupSuite() { ctx, flush := tester.NewContext(suite.T()) defer flush() - suite.connector = loadController(ctx, suite.T(), resource.Sites) + suite.connector = newController(ctx, suite.T(), resource.Sites) suite.user = tester.M365UserID(suite.T()) tester.LogTimeOfTest(suite.T()) @@ -362,7 +362,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { var ( siteID = tester.M365SiteID(t) - ctrl = loadController(ctx, t, resource.Sites) + ctrl = newController(ctx, t, resource.Sites) siteIDs = []string{siteID} ) @@ -409,7 +409,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { var ( siteID = tester.M365SiteID(t) - ctrl = loadController(ctx, t, resource.Sites) + ctrl = newController(ctx, t, resource.Sites) siteIDs = []string{siteID} ) diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index e7be0ae5f..2d63e4261 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -17,6 +17,7 @@ import ( exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" + "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -24,6 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" ) // --------------------------------------------------------------------------- @@ -280,7 +282,7 @@ func (suite *ControllerIntegrationSuite) SetupSuite() { ctx, flush := tester.NewContext(t) defer flush() - suite.ctrl = loadController(ctx, t, resource.Users) + suite.ctrl = newController(ctx, t, resource.Users) suite.user = tester.M365UserID(t) suite.secondaryUser = tester.SecondaryM365UserID(t) @@ -407,7 +409,7 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { func runRestore( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument - config ConfigInfo, + config stub.ConfigInfo, backupVersion int, collections []data.RestoreCollection, numRestoreItems int, @@ -419,7 +421,7 @@ func runRestore( start := time.Now() - restoreCtrl := loadController(ctx, t, config.Resource) + restoreCtrl := newController(ctx, t, config.Resource) restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true) deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, @@ -450,11 +452,11 @@ func runRestore( func runBackupAndCompare( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument - config ConfigInfo, + config stub.ConfigInfo, expectedData map[string]map[string][]byte, totalItems int, totalKopiaItems int, - inputCollections []ColInfo, + inputCollections []stub.ColInfo, ) { t.Helper() @@ -481,7 +483,7 @@ func runBackupAndCompare( nameToID[ro] = ro } - backupCtrl := loadController(ctx, t, config.Resource) + backupCtrl := newController(ctx, t, config.Resource) backupCtrl.IDNameLookup = inMock.NewCache(idToName, nameToID) backupSel := backupSelectorForExpected(t, config.Service, expectedDests) @@ -531,7 +533,7 @@ func runRestoreBackupTest( ctx, flush := tester.NewContext(t) defer flush() - config := ConfigInfo{ + config := stub.ConfigInfo{ Opts: opts, Resource: test.resourceCat, Service: test.service, @@ -540,7 +542,7 @@ func runRestoreBackupTest( RestoreCfg: tester.DefaultTestRestoreConfig(""), } - totalItems, totalKopiaItems, collections, expectedData, err := GetCollectionsAndExpected( + totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected( config, test.collections, version.Backup) @@ -576,16 +578,16 @@ func runRestoreTestWithVersion( ctx, flush := tester.NewContext(t) defer flush() - config := ConfigInfo{ + config := stub.ConfigInfo{ Opts: opts, - Resource: test.resource, + Resource: test.resourceCat, Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, RestoreCfg: tester.DefaultTestRestoreConfig(""), } - totalItems, _, collections, _, err := GetCollectionsAndExpected( + totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( config, test.collectionsPrevious, test.backupVersion) @@ -613,16 +615,16 @@ func runRestoreBackupTestVersions( ctx, flush := tester.NewContext(t) defer flush() - config := ConfigInfo{ + config := stub.ConfigInfo{ Opts: opts, - Resource: test.resource, + Resource: test.resourceCat, Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, RestoreCfg: tester.DefaultTestRestoreConfig(""), } - totalItems, _, collections, _, err := GetCollectionsAndExpected( + totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( config, test.collectionsPrevious, test.backupVersion) @@ -637,7 +639,7 @@ func runRestoreBackupTestVersions( totalItems) // Get expected output for new version. - totalItems, totalKopiaItems, _, expectedData, err := GetCollectionsAndExpected( + totalItems, totalKopiaItems, _, expectedData, err := stub.GetCollectionsAndExpected( config, test.collectionsLatest, version.Backup) @@ -662,24 +664,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "EmailsWithAttachments", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.MessageWithDirectAttachment( + Name: "someencodeditemID", + Data: exchMock.MessageWithDirectAttachment( subjectText + "-1", ), - lookupKey: subjectText + "-1", + LookupKey: subjectText + "-1", }, { - name: "someencodeditemID2", - data: exchMock.MessageWithTwoAttachments( + Name: "someencodeditemID2", + Data: exchMock.MessageWithTwoAttachments( subjectText + "-2", ), - lookupKey: subjectText + "-2", + LookupKey: subjectText + "-2", }, }, }, @@ -689,73 +691,73 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "MultipleEmailsMultipleFolders", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID", + Data: exchMock.MessageWithBodyBytes( subjectText+"-1", bodyText+" 1.", bodyText+" 1.", ), - lookupKey: subjectText + "-1", + LookupKey: subjectText + "-1", }, }, }, { PathElements: []string{"Work"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID2", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID2", + Data: exchMock.MessageWithBodyBytes( subjectText+"-2", bodyText+" 2.", bodyText+" 2.", ), - lookupKey: subjectText + "-2", + LookupKey: subjectText + "-2", }, { - name: "someencodeditemID3", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID3", + Data: exchMock.MessageWithBodyBytes( subjectText+"-3", bodyText+" 3.", bodyText+" 3.", ), - lookupKey: subjectText + "-3", + LookupKey: subjectText + "-3", }, }, }, { PathElements: []string{"Work", "Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID4", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID4", + Data: exchMock.MessageWithBodyBytes( subjectText+"-4", bodyText+" 4.", bodyText+" 4.", ), - lookupKey: subjectText + "-4", + LookupKey: subjectText + "-4", }, }, }, { PathElements: []string{"Work", "Inbox", "Work"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID5", - data: exchMock.MessageWithBodyBytes( + Name: "someencodeditemID5", + Data: exchMock.MessageWithBodyBytes( subjectText+"-5", bodyText+" 5.", bodyText+" 5.", ), - lookupKey: subjectText + "-5", + LookupKey: subjectText + "-5", }, }, }, @@ -765,25 +767,25 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "MultipleContactsSingleFolder", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Contacts"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.ContactBytes("Ghimley"), - lookupKey: "Ghimley", + Name: "someencodeditemID", + Data: exchMock.ContactBytes("Ghimley"), + LookupKey: "Ghimley", }, { - name: "someencodeditemID2", - data: exchMock.ContactBytes("Irgot"), - lookupKey: "Irgot", + Name: "someencodeditemID2", + Data: exchMock.ContactBytes("Irgot"), + LookupKey: "Irgot", }, { - name: "someencodeditemID3", - data: exchMock.ContactBytes("Jannes"), - lookupKey: "Jannes", + Name: "someencodeditemID3", + Data: exchMock.ContactBytes("Jannes"), + LookupKey: "Jannes", }, }, }, @@ -793,41 +795,41 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { name: "MultipleContactsMultipleFolders", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Work"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.ContactBytes("Ghimley"), - lookupKey: "Ghimley", + Name: "someencodeditemID", + Data: exchMock.ContactBytes("Ghimley"), + LookupKey: "Ghimley", }, { - name: "someencodeditemID2", - data: exchMock.ContactBytes("Irgot"), - lookupKey: "Irgot", + Name: "someencodeditemID2", + Data: exchMock.ContactBytes("Irgot"), + LookupKey: "Irgot", }, { - name: "someencodeditemID3", - data: exchMock.ContactBytes("Jannes"), - lookupKey: "Jannes", + Name: "someencodeditemID3", + Data: exchMock.ContactBytes("Jannes"), + LookupKey: "Jannes", }, }, }, { PathElements: []string{"Personal"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID4", - data: exchMock.ContactBytes("Argon"), - lookupKey: "Argon", + Name: "someencodeditemID4", + Data: exchMock.ContactBytes("Argon"), + LookupKey: "Argon", }, { - name: "someencodeditemID5", - data: exchMock.ContactBytes("Bernard"), - lookupKey: "Bernard", + Name: "someencodeditemID5", + Data: exchMock.ContactBytes("Bernard"), + LookupKey: "Bernard", }, }, }, @@ -926,26 +928,26 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { name: "Contacts", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Work"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID", - data: exchMock.ContactBytes("Ghimley"), - lookupKey: "Ghimley", + Name: "someencodeditemID", + Data: exchMock.ContactBytes("Ghimley"), + LookupKey: "Ghimley", }, }, }, { PathElements: []string{"Personal"}, Category: path.ContactsCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "someencodeditemID2", - data: exchMock.ContactBytes("Irgot"), - lookupKey: "Irgot", + Name: "someencodeditemID2", + Data: exchMock.ContactBytes("Irgot"), + LookupKey: "Irgot", }, }, }, @@ -1004,12 +1006,12 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { }, }) - totalItems, _, collections, expectedData, err := collectionsForInfo( + totalItems, _, collections, expectedData, err := stub.CollectionsForInfo( test.service, suite.ctrl.tenant, suite.user, restoreCfg, - []ColInfo{collection}, + []stub.ColInfo{collection}, version.Backup, ) require.NoError(t, err) @@ -1027,7 +1029,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { restoreCfg.Location, ) - restoreCtrl := loadController(ctx, t, test.resourceCat) + restoreCtrl := newController(ctx, t, test.resourceCat) deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, version.Backup, @@ -1057,7 +1059,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { // Run a backup and compare its output with what we put in. - backupCtrl := loadController(ctx, t, test.resourceCat) + backupCtrl := newController(ctx, t, test.resourceCat) backupSel := backupSelectorForExpected(t, test.service, expectedDests) t.Log("Selective backup of", backupSel) @@ -1079,7 +1081,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { t.Log("Backup enumeration complete") - ci := ConfigInfo{ + ci := stub.ConfigInfo{ Opts: control.Options{RestorePermissions: true}, // Alright to be empty, needed for OneDrive. RestoreCfg: control.RestoreConfig{}, @@ -1105,15 +1107,15 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen name: "EmailsWithLargeAttachments", service: path.ExchangeService, resourceCat: resource.Users, - collections: []ColInfo{ + collections: []stub.ColInfo{ { PathElements: []string{"Inbox"}, Category: path.EmailCategory, - Items: []ItemInfo{ + Items: []stub.ItemInfo{ { - name: "35mbAttachment", - data: exchMock.MessageWithSizedAttachment(subjectText, 35), - lookupKey: subjectText, + Name: "35mbAttachment", + Data: exchMock.MessageWithSizedAttachment(subjectText, 35), + LookupKey: subjectText, }, }, }, @@ -1206,7 +1208,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { defer flush() var ( - backupCtrl = loadController(ctx, t, test.resourceCat) + backupCtrl = newController(ctx, t, test.resourceCat) backupSel = test.selectorFunc(t) errs = fault.New(true) start = time.Now() @@ -1270,3 +1272,166 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { }) } } + +type DisconnectedUnitSuite struct { + tester.Suite +} + +func TestDisconnectedUnitSuite(t *testing.T) { + s := &DisconnectedUnitSuite{ + Suite: tester.NewUnitSuite(t), + } + + suite.Run(t, s) +} + +func statusTestTask( + t *testing.T, + ctrl *Controller, + objects, success, folder int, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + status := support.CreateStatus( + ctx, + support.Restore, folder, + support.CollectionMetrics{ + Objects: objects, + Successes: success, + Bytes: 0, + }, + "statusTestTask") + ctrl.UpdateStatus(status) +} + +func (suite *DisconnectedUnitSuite) TestController_Status() { + t := suite.T() + ctrl := Controller{wg: &sync.WaitGroup{}} + + // Two tasks + ctrl.incrementAwaitingMessages() + ctrl.incrementAwaitingMessages() + + // Each helper task processes 4 objects, 1 success, 3 errors, 1 folders + go statusTestTask(t, &ctrl, 4, 1, 1) + go statusTestTask(t, &ctrl, 4, 1, 1) + + stats := ctrl.Wait() + + assert.NotEmpty(t, ctrl.PrintableStatus()) + // Expect 8 objects + assert.Equal(t, 8, stats.Objects) + // Expect 2 success + assert.Equal(t, 2, stats.Successes) + // Expect 2 folders + assert.Equal(t, 2, stats.Folders) +} + +func (suite *DisconnectedUnitSuite) TestVerifyBackupInputs_allServices() { + sites := []string{"abc.site.foo", "bar.site.baz"} + + tests := []struct { + name string + excludes func(t *testing.T) selectors.Selector + filters func(t *testing.T) selectors.Selector + includes func(t *testing.T) selectors.Selector + checkError assert.ErrorAssertionFunc + }{ + { + name: "Valid User", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = "elliotReid@someHospital.org" + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = "elliotReid@someHospital.org" + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = "elliotReid@someHospital.org" + return sel.Selector + }, + }, + { + name: "Invalid User", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + return sel.Selector + }, + }, + { + name: "valid sites", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) + sel.DiscreteOwner = "abc.site.foo" + sel.Exclude(sel.AllData()) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) + sel.DiscreteOwner = "abc.site.foo" + sel.Filter(sel.AllData()) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) + sel.DiscreteOwner = "abc.site.foo" + sel.Include(sel.AllData()) + return sel.Selector + }, + }, + { + name: "invalid sites", + checkError: assert.Error, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) + sel.Exclude(sel.AllData()) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) + sel.Filter(sel.AllData()) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) + sel.Include(sel.AllData()) + return sel.Selector + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + err := verifyBackupInputs(test.excludes(t), sites) + test.checkError(t, err, clues.ToCore(err)) + err = verifyBackupInputs(test.filters(t), sites) + test.checkError(t, err, clues.ToCore(err)) + err = verifyBackupInputs(test.includes(t), sites) + test.checkError(t, err, clues.ToCore(err)) + }) + } +} diff --git a/src/internal/m365/exchange/contact_container_cache.go b/src/internal/m365/exchange/contacts_container_cache.go similarity index 100% rename from src/internal/m365/exchange/contact_container_cache.go rename to src/internal/m365/exchange/contacts_container_cache.go diff --git a/src/internal/m365/exchange/event_container_cache.go b/src/internal/m365/exchange/events_container_cache.go similarity index 100% rename from src/internal/m365/exchange/event_container_cache.go rename to src/internal/m365/exchange/events_container_cache.go diff --git a/src/internal/m365/graph_connector_disconnected_test.go b/src/internal/m365/graph_connector_disconnected_test.go deleted file mode 100644 index c2dee9d58..000000000 --- a/src/internal/m365/graph_connector_disconnected_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package m365 - -import ( - "sync" - "testing" - - "github.com/alcionai/clues" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/internal/m365/support" - "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/selectors" - selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" -) - -// --------------------------------------------------------------- -// Disconnected Test Section -// --------------------------------------------------------------- -type DisconnectedSuite struct { - tester.Suite -} - -func TestSuite(t *testing.T) { - s := &DisconnectedSuite{ - Suite: tester.NewUnitSuite(t), - } - - suite.Run(t, s) -} - -func statusTestTask( - t *testing.T, - ctrl *Controller, - objects, success, folder int, -) { - ctx, flush := tester.NewContext(t) - defer flush() - - status := support.CreateStatus( - ctx, - support.Restore, folder, - support.CollectionMetrics{ - Objects: objects, - Successes: success, - Bytes: 0, - }, - "statusTestTask") - ctrl.UpdateStatus(status) -} - -func (suite *DisconnectedSuite) TestController_Status() { - t := suite.T() - ctrl := Controller{wg: &sync.WaitGroup{}} - - // Two tasks - ctrl.incrementAwaitingMessages() - ctrl.incrementAwaitingMessages() - - // Each helper task processes 4 objects, 1 success, 3 errors, 1 folders - go statusTestTask(t, &ctrl, 4, 1, 1) - go statusTestTask(t, &ctrl, 4, 1, 1) - - stats := ctrl.Wait() - - assert.NotEmpty(t, ctrl.PrintableStatus()) - // Expect 8 objects - assert.Equal(t, 8, stats.Objects) - // Expect 2 success - assert.Equal(t, 2, stats.Successes) - // Expect 2 folders - assert.Equal(t, 2, stats.Folders) -} - -func (suite *DisconnectedSuite) TestVerifyBackupInputs_allServices() { - sites := []string{"abc.site.foo", "bar.site.baz"} - - tests := []struct { - name string - excludes func(t *testing.T) selectors.Selector - filters func(t *testing.T) selectors.Selector - includes func(t *testing.T) selectors.Selector - checkError assert.ErrorAssertionFunc - }{ - { - name: "Valid User", - checkError: assert.NoError, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = "elliotReid@someHospital.org" - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Filter(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = "elliotReid@someHospital.org" - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Include(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = "elliotReid@someHospital.org" - return sel.Selector - }, - }, - { - name: "Invalid User", - checkError: assert.NoError, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Filter(selTD.OneDriveBackupFolderScope(sel)) - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Include(selTD.OneDriveBackupFolderScope(sel)) - return sel.Selector - }, - }, - { - name: "valid sites", - checkError: assert.NoError, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) - sel.DiscreteOwner = "abc.site.foo" - sel.Exclude(sel.AllData()) - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) - sel.DiscreteOwner = "abc.site.foo" - sel.Filter(sel.AllData()) - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"}) - sel.DiscreteOwner = "abc.site.foo" - sel.Include(sel.AllData()) - return sel.Selector - }, - }, - { - name: "invalid sites", - checkError: assert.Error, - excludes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) - sel.Exclude(sel.AllData()) - return sel.Selector - }, - filters: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) - sel.Filter(sel.AllData()) - return sel.Selector - }, - includes: func(t *testing.T) selectors.Selector { - sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"}) - sel.Include(sel.AllData()) - return sel.Selector - }, - }, - } - - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - err := verifyBackupInputs(test.excludes(t), sites) - test.checkError(t, err, clues.ToCore(err)) - err = verifyBackupInputs(test.filters(t), sites) - test.checkError(t, err, clues.ToCore(err)) - err = verifyBackupInputs(test.includes(t), sites) - test.checkError(t, err, clues.ToCore(err)) - }) - } -} diff --git a/src/internal/m365/graph_connector_helper_test.go b/src/internal/m365/helper_test.go similarity index 98% rename from src/internal/m365/graph_connector_helper_test.go rename to src/internal/m365/helper_test.go index 1fc1573c7..fe1de81f8 100644 --- a/src/internal/m365/graph_connector_helper_test.go +++ b/src/internal/m365/helper_test.go @@ -19,7 +19,9 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" + m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -102,15 +104,15 @@ func testElementsMatch[T any]( type restoreBackupInfo struct { name string service path.ServiceType - collections []ColInfo + collections []m365Stub.ColInfo resourceCat resource.Category } type restoreBackupInfoMultiVersion struct { service path.ServiceType - collectionsLatest []ColInfo - collectionsPrevious []ColInfo - resource resource.Category + collectionsLatest []m365Stub.ColInfo + collectionsPrevious []m365Stub.ColInfo + resourceCat resource.Category backupVersion int } @@ -686,7 +688,7 @@ func compareDriveItem( t *testing.T, expected map[string][]byte, item data.Stream, - config ConfigInfo, + config m365Stub.ConfigInfo, rootDir bool, ) bool { // Skip Drive permissions in the folder that used to be the root. We don't @@ -793,7 +795,7 @@ func compareDriveItem( return true } - var fileData testOneDriveData + var fileData odStub.FileData err = json.Unmarshal(buf, &fileData) if !assert.NoError(t, err, "unmarshalling file data for file", name, clues.ToCore(err)) { @@ -829,7 +831,7 @@ func compareItem( service path.ServiceType, category path.CategoryType, item data.Stream, - config ConfigInfo, + config m365Stub.ConfigInfo, rootDir bool, ) bool { if mt, ok := item.(data.StreamModTime); ok { @@ -923,7 +925,7 @@ func checkCollections( expectedItems int, expected map[string]map[string][]byte, got []data.BackupCollection, - config ConfigInfo, + config m365Stub.ConfigInfo, ) int { collectionsWithItems := []data.BackupCollection{} @@ -985,7 +987,7 @@ func checkCollections( checkHasCollections(t, expected, collectionsWithItems) // Return how many metadata files were skipped so we can account for it in the - // check on Controller status. + // check on controller status. return skipped } @@ -1152,11 +1154,11 @@ func getSelectorWith( } } -func loadController(ctx context.Context, t *testing.T, r resource.Category) *Controller { +func newController(ctx context.Context, t *testing.T, r resource.Category) *Controller { a := tester.NewM365Account(t) - connector, err := NewController(ctx, a, r) + controller, err := NewController(ctx, a, r) require.NoError(t, err, clues.ToCore(err)) - return connector + return controller } diff --git a/src/internal/m365/mock/collection.go b/src/internal/m365/mock/collection.go new file mode 100644 index 000000000..e1d61db58 --- /dev/null +++ b/src/internal/m365/mock/collection.go @@ -0,0 +1,24 @@ +package mock + +import ( + "context" + + "github.com/alcionai/corso/src/internal/data" +) + +type RestoreCollection struct { + data.Collection + AuxItems map[string]data.Stream +} + +func (rc RestoreCollection) FetchItemByName( + ctx context.Context, + name string, +) (data.Stream, error) { + res := rc.AuxItems[name] + if res == nil { + return nil, data.ErrNotFound + } + + return res, nil +} diff --git a/src/internal/m365/onedrive/drive.go b/src/internal/m365/onedrive/item_collector.go similarity index 100% rename from src/internal/m365/onedrive/drive.go rename to src/internal/m365/onedrive/item_collector.go diff --git a/src/internal/m365/onedrive/drive_test.go b/src/internal/m365/onedrive/item_collector_test.go similarity index 98% rename from src/internal/m365/onedrive/drive_test.go rename to src/internal/m365/onedrive/item_collector_test.go index 348722b78..65e9bf5fe 100644 --- a/src/internal/m365/onedrive/drive_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -28,13 +28,12 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) -// Unit tests -type OneDriveUnitSuite struct { +type ItemCollectorUnitSuite struct { tester.Suite } func TestOneDriveUnitSuite(t *testing.T) { - suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)}) + suite.Run(t, &ItemCollectorUnitSuite{Suite: tester.NewUnitSuite(t)}) } const ( @@ -51,7 +50,7 @@ func odErr(code string) *odataerrors.ODataError { return odErr } -func (suite *OneDriveUnitSuite) TestDrives() { +func (suite *ItemCollectorUnitSuite) TestDrives() { t := suite.T() ctx, flush := tester.NewContext(t) diff --git a/src/internal/m365/graph_connector_onedrive_test_helper.go b/src/internal/m365/onedrive/stub/stub.go similarity index 59% rename from src/internal/m365/graph_connector_onedrive_test_helper.go rename to src/internal/m365/onedrive/stub/stub.go index 77acc1b7d..44590d5a1 100644 --- a/src/internal/m365/graph_connector_onedrive_test_helper.go +++ b/src/internal/m365/onedrive/stub/stub.go @@ -1,4 +1,4 @@ -package m365 +package stub import ( "encoding/json" @@ -6,11 +6,10 @@ import ( "github.com/alcionai/clues" "github.com/google/uuid" - "golang.org/x/exp/maps" - "github.com/alcionai/corso/src/internal/data" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" ) @@ -61,59 +60,59 @@ type ItemData struct { Perms PermData } -type OnedriveColInfo struct { +type ColInfo struct { PathElements []string Perms PermData Files []ItemData Folders []ItemData } -type onedriveCollection struct { - service path.ServiceType +type collection struct { + Service path.ServiceType PathElements []string - items []ItemInfo - aux []ItemInfo - backupVersion int + Items []m365Stub.ItemInfo + Aux []m365Stub.ItemInfo + BackupVersion int } -func (c onedriveCollection) collection() ColInfo { +func (c collection) ColInfo() m365Stub.ColInfo { cat := path.FilesCategory - if c.service == path.SharePointService { + if c.Service == path.SharePointService { cat = path.LibrariesCategory } - return ColInfo{ + return m365Stub.ColInfo{ PathElements: c.PathElements, Category: cat, - Items: c.items, - AuxItems: c.aux, + Items: c.Items, + AuxItems: c.Aux, } } -func NewOneDriveCollection( +func NewCollection( service path.ServiceType, PathElements []string, backupVersion int, -) *onedriveCollection { - return &onedriveCollection{ - service: service, +) *collection { + return &collection{ + Service: service, PathElements: PathElements, - backupVersion: backupVersion, + BackupVersion: backupVersion, } } func DataForInfo( service path.ServiceType, - cols []OnedriveColInfo, + cols []ColInfo, backupVersion int, -) ([]ColInfo, error) { +) ([]m365Stub.ColInfo, error) { var ( - res []ColInfo + res []m365Stub.ColInfo err error ) for _, c := range cols { - onedriveCol := NewOneDriveCollection(service, c.PathElements, backupVersion) + onedriveCol := NewCollection(service, c.PathElements, backupVersion) for _, f := range c.Files { _, err = onedriveCol.withFile(f.Name, f.Data, f.Perms) @@ -134,18 +133,18 @@ func DataForInfo( return res, err } - res = append(res, onedriveCol.collection()) + res = append(res, onedriveCol.ColInfo()) } return res, nil } -func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermData) (*onedriveCollection, error) { - switch c.backupVersion { +func (c *collection) withFile(name string, fileData []byte, perm PermData) (*collection, error) { + switch c.BackupVersion { case 0: // Lookups will occur using the most recent version of things so we need // the embedded file name to match that. - item, err := onedriveItemWithData( + item, err := FileWithData( name, name+metadata.DataFileSuffix, fileData) @@ -153,12 +152,12 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat return c, err } - c.items = append(c.items, item) + c.Items = append(c.Items, item) // v1-5, early metadata design case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName: - items, err := onedriveItemWithData( + items, err := FileWithData( name+metadata.DataFileSuffix, name+metadata.DataFileSuffix, fileData) @@ -166,24 +165,24 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat return c, err } - c.items = append(c.items, items) + c.Items = append(c.Items, items) - md, err := onedriveMetadata( + md, err := ItemWithMetadata( "", name+metadata.MetaFileSuffix, name+metadata.MetaFileSuffix, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) if err != nil { return c, err } - c.items = append(c.items, md) - c.aux = append(c.aux, md) + c.Items = append(c.Items, md) + c.Aux = append(c.Aux, md) // v6+ current metadata design case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID: - item, err := onedriveItemWithData( + item, err := FileWithData( name+metadata.DataFileSuffix, name+metadata.DataFileSuffix, fileData) @@ -191,50 +190,50 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat return c, err } - c.items = append(c.items, item) + c.Items = append(c.Items, item) - md, err := onedriveMetadata( + md, err := ItemWithMetadata( name, name+metadata.MetaFileSuffix, name, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) if err != nil { return c, err } - c.items = append(c.items, md) - c.aux = append(c.aux, md) + c.Items = append(c.Items, md) + c.Aux = append(c.Aux, md) default: - return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.backupVersion)) + return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.BackupVersion)) } return c, nil } -func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCollection, error) { - switch c.backupVersion { +func (c *collection) withFolder(name string, perm PermData) (*collection, error) { + switch c.BackupVersion { case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName, version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID: return c, nil case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker: - item, err := onedriveMetadata( + item, err := ItemWithMetadata( "", name+metadata.DirMetaFileSuffix, name+metadata.DirMetaFileSuffix, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, item) + c.Items = append(c.Items, item) if err != nil { return c, err } default: - return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.backupVersion)) + return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.BackupVersion)) } return c, nil @@ -242,17 +241,17 @@ func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCo // withPermissions adds permissions to the folder represented by this // onedriveCollection. -func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection, error) { +func (c *collection) withPermissions(perm PermData) (*collection, error) { // These versions didn't store permissions for the folder or didn't store them // in the folder's collection. - if c.backupVersion < version.OneDrive4DirIncludesPermissions { + if c.BackupVersion < version.OneDrive4DirIncludesPermissions { return c, nil } name := c.PathElements[len(c.PathElements)-1] metaName := name - if c.backupVersion >= version.OneDrive5DirMetaNoName { + if c.BackupVersion >= version.OneDrive5DirMetaNoName { // We switched to just .dirmeta for metadata file names. metaName = "" } @@ -261,98 +260,63 @@ func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection return c, nil } - md, err := onedriveMetadata( + md, err := ItemWithMetadata( name, metaName+metadata.DirMetaFileSuffix, metaName+metadata.DirMetaFileSuffix, perm, - c.backupVersion >= versionPermissionSwitchedToID) + c.BackupVersion >= versionPermissionSwitchedToID) if err != nil { return c, err } - c.items = append(c.items, md) - c.aux = append(c.aux, md) + c.Items = append(c.Items, md) + c.Aux = append(c.Aux, md) return c, err } -type testOneDriveData struct { +type FileData struct { FileName string `json:"fileName,omitempty"` Data []byte `json:"data,omitempty"` } -func onedriveItemWithData( +func FileWithData( name, lookupKey string, fileData []byte, -) (ItemInfo, error) { - content := testOneDriveData{ +) (m365Stub.ItemInfo, error) { + content := FileData{ FileName: lookupKey, Data: fileData, } serialized, err := json.Marshal(content) if err != nil { - return ItemInfo{}, clues.Stack(err) + return m365Stub.ItemInfo{}, clues.Stack(err) } - return ItemInfo{ - name: name, - data: serialized, - lookupKey: lookupKey, + return m365Stub.ItemInfo{ + Name: name, + Data: serialized, + LookupKey: lookupKey, }, nil } -func onedriveMetadata( +func ItemWithMetadata( fileName, itemID, lookupKey string, perm PermData, permUseID bool, -) (ItemInfo, error) { +) (m365Stub.ItemInfo, error) { testMeta := getMetadata(fileName, perm, permUseID) testMetaJSON, err := json.Marshal(testMeta) if err != nil { - return ItemInfo{}, clues.Wrap(err, "marshalling metadata") + return m365Stub.ItemInfo{}, clues.Wrap(err, "marshalling metadata") } - return ItemInfo{ - name: itemID, - data: testMetaJSON, - lookupKey: lookupKey, + return m365Stub.ItemInfo{ + Name: itemID, + Data: testMetaJSON, + LookupKey: lookupKey, }, nil } - -func GetCollectionsAndExpected( - config ConfigInfo, - testCollections []ColInfo, - backupVersion int, -) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { - var ( - collections []data.RestoreCollection - expectedData = map[string]map[string][]byte{} - totalItems = 0 - totalKopiaItems = 0 - ) - - for _, owner := range config.ResourceOwners { - numItems, kopiaItems, ownerCollections, userExpectedData, err := collectionsForInfo( - config.Service, - config.Tenant, - owner, - config.RestoreCfg, - testCollections, - backupVersion, - ) - if err != nil { - return totalItems, totalKopiaItems, collections, expectedData, err - } - - collections = append(collections, ownerCollections...) - totalItems += numItems - totalKopiaItems += kopiaItems - - maps.Copy(expectedData, userExpectedData) - } - - return totalItems, totalKopiaItems, collections, expectedData, nil -} diff --git a/src/internal/m365/graph_connector_onedrive_test.go b/src/internal/m365/onedrive_test.go similarity index 88% rename from src/internal/m365/graph_connector_onedrive_test.go rename to src/internal/m365/onedrive_test.go index 1a45992d3..ca821392f 100644 --- a/src/internal/m365/graph_connector_onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -96,17 +97,17 @@ type oneDriveSuite interface { } type suiteInfoImpl struct { - ac api.Client - controller *Controller - resourceOwner string - resourceCat resource.Category - secondaryUser string - secondaryUserID string - service path.ServiceType - tertiaryUser string - tertiaryUserID string - user string - userID string + ac api.Client + controller *Controller + resourceOwner string + resourceCategory resource.Category + secondaryUser string + secondaryUserID string + service path.ServiceType + tertiaryUser string + tertiaryUserID string + user string + userID string } func NewSuiteInfoImpl( @@ -115,22 +116,22 @@ func NewSuiteInfoImpl( resourceOwner string, service path.ServiceType, ) suiteInfoImpl { - rc := resource.Users + rsc := resource.Users if service == path.SharePointService { - rc = resource.Sites + rsc = resource.Sites } - ctrl := loadController(ctx, t, rc) + ctrl := newController(ctx, t, rsc) return suiteInfoImpl{ - ac: ctrl.AC, - controller: ctrl, - resourceOwner: resourceOwner, - resourceCat: rc, - secondaryUser: tester.SecondaryM365UserID(t), - service: service, - tertiaryUser: tester.TertiaryM365UserID(t), - user: tester.M365UserID(t), + ac: ctrl.AC, + controller: ctrl, + resourceOwner: resourceOwner, + resourceCategory: rsc, + secondaryUser: tester.SecondaryM365UserID(t), + service: service, + tertiaryUser: tester.TertiaryM365UserID(t), + user: tester.M365UserID(t), } } @@ -163,7 +164,7 @@ func (si suiteInfoImpl) Service() path.ServiceType { } func (si suiteInfoImpl) Resource() resource.Category { - return si.resourceCat + return si.resourceCategory } // --------------------------------------------------------------------------- @@ -402,16 +403,16 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( folderBName, } - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderAName, }, @@ -422,13 +423,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: folderAPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileBData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderBName, }, @@ -436,13 +437,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: subfolderBPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileCData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderAName, }, @@ -450,7 +451,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: subfolderAPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileDData, @@ -459,7 +460,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, { PathElements: folderBPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileEData, @@ -468,18 +469,18 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) for vn := startVersion; vn <= version.Backup; vn++ { suite.Run(fmt.Sprintf("Version%d", vn), func() { t := suite.T() - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -549,15 +550,15 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { folderCName, } - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{ + Files: []stub.ItemData{ { // Test restoring a file that doesn't inherit permissions. Name: fileName, Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -570,13 +571,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { Data: fileBData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderBName, }, { Name: folderAName, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -584,7 +585,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { }, { Name: folderCName, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -594,23 +595,23 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { }, { PathElements: folderBPath, - Files: []ItemData{ + Files: []stub.ItemData{ { // Test restoring a file in a non-root folder that doesn't inherit // permissions. Name: fileName, Data: fileBData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderAName, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -624,18 +625,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // // Tests a folder that has permissions with an item in the folder with // // the same permissions. // pathElements: subfolderAPath, - // files: []itemData{ + // files: []stub.ItemData{ // { // name: fileName, // data: fileDData, - // perms: permData{ + // perms: stub.PermData{ // user: secondaryUserName, // entityID: secondaryUserID, // roles: readPerm, // }, // }, // }, - // Perms: PermData{ + // Perms: stub.PermData{ // User: secondaryUserName, // EntityID: secondaryUserID, // Roles: readPerm, @@ -645,18 +646,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // Tests a folder that has permissions with an item in the folder with // the different permissions. PathElements: folderAPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileEData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, }, }, }, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -666,13 +667,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // Tests a folder that has permissions with an item in the folder with // no permissions. PathElements: folderCPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, }, }, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: readPerm, @@ -680,7 +681,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() @@ -690,12 +691,12 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { // Ideally this can always be true or false and still // work, but limiting older versions to use emails so as // to validate that flow as well. - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -730,18 +731,18 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { suite.Service(), suite.ResourceOwner()) - inputCols := []OnedriveColInfo{ + inputCols := []stub.ColInfo{ { PathElements: []string{ odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, }, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -751,14 +752,14 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { }, } - expectedCols := []OnedriveColInfo{ + expectedCols := []stub.ColInfo{ { PathElements: []string{ odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, }, - Files: []ItemData{ + Files: []stub.ItemData{ { // No permissions on the output since they weren't restored. Name: fileName, @@ -768,19 +769,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { }, } - expected, err := DataForInfo(suite.Service(), expectedCols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), expectedCols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() for vn := startVersion; vn <= version.Backup; vn++ { suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() { t := suite.T() - input, err := DataForInfo(suite.Service(), inputCols, vn) + input, err := stub.DataForInfo(suite.Service(), inputCols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -855,11 +856,11 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio folderCName, } - fileSet := []ItemData{ + fileSet := []stub.ItemData{ { Name: "file-custom", Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ User: secondaryUserName, EntityID: secondaryUserID, Roles: writePerm, @@ -869,14 +870,14 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio { Name: "file-inherited", Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeInherited, }, }, { Name: "file-empty", Data: fileAData, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeCustom, }, }, @@ -900,23 +901,23 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio // - inherted-permission-file // - empty-permission-file (empty/empty might have interesting behavior) - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{}, - Folders: []ItemData{ + Files: []stub.ItemData{}, + Folders: []stub.ItemData{ {Name: folderAName}, }, }, { PathElements: folderAPath, Files: fileSet, - Folders: []ItemData{ + Folders: []stub.ItemData{ {Name: folderAName}, {Name: folderBName}, {Name: folderCName}, }, - Perms: PermData{ + Perms: stub.PermData{ User: tertiaryUserName, EntityID: tertiaryUserID, Roles: readPerm, @@ -925,7 +926,7 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio { PathElements: subfolderAAPath, Files: fileSet, - Perms: PermData{ + Perms: stub.PermData{ User: tertiaryUserName, EntityID: tertiaryUserID, Roles: writePerm, @@ -935,20 +936,20 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio { PathElements: subfolderABPath, Files: fileSet, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeInherited, }, }, { PathElements: subfolderACPath, Files: fileSet, - Perms: PermData{ + Perms: stub.PermData{ SharingMode: metadata.SharingModeCustom, }, }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() @@ -958,12 +959,12 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio // Ideally this can always be true or false and still // work, but limiting older versions to use emails so as // to validate that flow as well. - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, @@ -1018,16 +1019,16 @@ func testRestoreFolderNamedFolderRegression( folderBName, } - cols := []OnedriveColInfo{ + cols := []stub.ColInfo{ { PathElements: rootPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileAData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderNamedFolder, }, @@ -1038,13 +1039,13 @@ func testRestoreFolderNamedFolderRegression( }, { PathElements: folderFolderPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileBData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderBName, }, @@ -1052,13 +1053,13 @@ func testRestoreFolderNamedFolderRegression( }, { PathElements: subfolderPath, - Files: []ItemData{ + Files: []stub.ItemData{ { Name: fileName, Data: fileCData, }, }, - Folders: []ItemData{ + Folders: []stub.ItemData{ { Name: folderNamedFolder, }, @@ -1066,19 +1067,19 @@ func testRestoreFolderNamedFolderRegression( }, } - expected, err := DataForInfo(suite.Service(), cols, version.Backup) + expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup) require.NoError(suite.T(), err) bss := suite.Service().String() for vn := startVersion; vn <= version.Backup; vn++ { suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() { t := suite.T() - input, err := DataForInfo(suite.Service(), cols, vn) + input, err := stub.DataForInfo(suite.Service(), cols, vn) require.NoError(suite.T(), err) testData := restoreBackupInfoMultiVersion{ service: suite.Service(), - resource: suite.Resource(), + resourceCat: suite.Resource(), backupVersion: vn, collectionsPrevious: input, collectionsLatest: expected, diff --git a/src/internal/m365/sharepoint/api/pages.go b/src/internal/m365/sharepoint/api/pages.go index 581d5de10..e7d24fd6f 100644 --- a/src/internal/m365/sharepoint/api/pages.go +++ b/src/internal/m365/sharepoint/api/pages.go @@ -172,7 +172,7 @@ func RestoreSitePage( itemData data.Stream, siteID, destName string, ) (details.ItemInfo, error) { - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID())) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID())) defer end() var ( diff --git a/src/internal/m365/sharepoint/collection.go b/src/internal/m365/sharepoint/collection.go index 90af58cbf..687a2ebb8 100644 --- a/src/internal/m365/sharepoint/collection.go +++ b/src/internal/m365/sharepoint/collection.go @@ -257,7 +257,7 @@ func (sc *Collection) retrieveLists( sc.data <- &Item{ id: ptr.Val(lst.GetId()), data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointListInfo(lst, size), + info: listToSPInfo(lst, size), modTime: t, } @@ -320,7 +320,7 @@ func (sc *Collection) retrievePages( sc.data <- &Item{ id: ptr.Val(pg.GetId()), data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointPageInfo(pg, root, size), + info: pageToSPInfo(pg, root, size), modTime: ptr.OrNow(pg.GetLastModifiedDateTime()), } diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/sharepoint/collection_test.go index 5381710c2..5177ecf06 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/sharepoint/collection_test.go @@ -116,7 +116,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { data := &Item{ id: name, data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointListInfo(listing, int64(len(byteArray))), + info: listToSPInfo(listing, int64(len(byteArray))), } return data @@ -205,7 +205,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { listData := &Item{ id: testName, data: io.NopCloser(bytes.NewReader(byteArray)), - info: sharePointListInfo(listing, int64(len(byteArray))), + info: listToSPInfo(listing, int64(len(byteArray))), } destName := tester.DefaultTestRestoreConfig("").Location diff --git a/src/internal/m365/sharepoint/list_info.go b/src/internal/m365/sharepoint/list_info.go deleted file mode 100644 index 62101b584..000000000 --- a/src/internal/m365/sharepoint/list_info.go +++ /dev/null @@ -1,28 +0,0 @@ -package sharepoint - -import ( - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/pkg/backup/details" -) - -// sharePointListInfo translates models.Listable metadata into searchable content -// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 -func sharePointListInfo(lst models.Listable, size int64) *details.SharePointInfo { - var ( - name = ptr.Val(lst.GetDisplayName()) - webURL = ptr.Val(lst.GetWebUrl()) - created = ptr.Val(lst.GetCreatedDateTime()) - modified = ptr.Val(lst.GetLastModifiedDateTime()) - ) - - return &details.SharePointInfo{ - ItemType: details.SharePointList, - ItemName: name, - Created: created, - Modified: modified, - WebURL: webURL, - Size: size, - } -} diff --git a/src/internal/m365/sharepoint/list_info_test.go b/src/internal/m365/sharepoint/list_info_test.go deleted file mode 100644 index a4467450f..000000000 --- a/src/internal/m365/sharepoint/list_info_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package sharepoint - -import ( - "testing" - - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/backup/details" -) - -type SharePointInfoSuite struct { - tester.Suite -} - -func TestSharePointInfoSuite(t *testing.T) { - suite.Run(t, &SharePointInfoSuite{Suite: tester.NewUnitSuite(t)}) -} - -func (suite *SharePointInfoSuite) TestSharePointInfo() { - tests := []struct { - name string - listAndDeets func() (models.Listable, *details.SharePointInfo) - }{ - { - name: "Empty List", - listAndDeets: func() (models.Listable, *details.SharePointInfo) { - i := &details.SharePointInfo{ItemType: details.SharePointList} - return models.NewList(), i - }, - }, { - name: "Only Name", - listAndDeets: func() (models.Listable, *details.SharePointInfo) { - aTitle := "Whole List" - listing := models.NewList() - listing.SetDisplayName(&aTitle) - i := &details.SharePointInfo{ - ItemType: details.SharePointList, - ItemName: aTitle, - } - - return listing, i - }, - }, - } - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - list, expected := test.listAndDeets() - info := sharePointListInfo(list, 10) - assert.Equal(t, expected.ItemType, info.ItemType) - assert.Equal(t, expected.ItemName, info.ItemName) - assert.Equal(t, expected.WebURL, info.WebURL) - }) - } -} diff --git a/src/internal/m365/sharepoint/list.go b/src/internal/m365/sharepoint/lists.go similarity index 93% rename from src/internal/m365/sharepoint/list.go rename to src/internal/m365/sharepoint/lists.go index 3dcaaa58f..0555516af 100644 --- a/src/internal/m365/sharepoint/list.go +++ b/src/internal/m365/sharepoint/lists.go @@ -10,9 +10,30 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" ) +// listToSPInfo translates models.Listable metadata into searchable content +// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 +func listToSPInfo(lst models.Listable, size int64) *details.SharePointInfo { + var ( + name = ptr.Val(lst.GetDisplayName()) + webURL = ptr.Val(lst.GetWebUrl()) + created = ptr.Val(lst.GetCreatedDateTime()) + modified = ptr.Val(lst.GetLastModifiedDateTime()) + ) + + return &details.SharePointInfo{ + ItemType: details.SharePointList, + ItemName: name, + Created: created, + Modified: modified, + WebURL: webURL, + Size: size, + } +} + type listTuple struct { name string id string diff --git a/src/internal/m365/sharepoint/list_test.go b/src/internal/m365/sharepoint/lists_test.go similarity index 53% rename from src/internal/m365/sharepoint/list_test.go rename to src/internal/m365/sharepoint/lists_test.go index 05990712d..32cb16918 100644 --- a/src/internal/m365/sharepoint/list_test.go +++ b/src/internal/m365/sharepoint/lists_test.go @@ -4,21 +4,23 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" ) -type SharePointSuite struct { +type ListsUnitSuite struct { tester.Suite creds account.M365Config } -func (suite *SharePointSuite) SetupSuite() { +func (suite *ListsUnitSuite) SetupSuite() { t := suite.T() a := tester.NewM365Account(t) m365, err := a.M365Config() @@ -27,8 +29,8 @@ func (suite *SharePointSuite) SetupSuite() { suite.creds = m365 } -func TestSharePointSuite(t *testing.T) { - suite.Run(t, &SharePointSuite{ +func TestListsUnitSuite(t *testing.T) { + suite.Run(t, &ListsUnitSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tester.M365AcctCredEnvs}, @@ -47,7 +49,7 @@ func TestSharePointSuite(t *testing.T) { // to verify if these 2 calls are valid // - fetchContentBaseTypes // - fetchColumnPositions -func (suite *SharePointSuite) TestLoadList() { +func (suite *ListsUnitSuite) TestLoadList() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -63,3 +65,42 @@ func (suite *SharePointSuite) TestLoadList() { assert.Greater(t, len(lists), 0) t.Logf("Length: %d\n", len(lists)) } + +func (suite *ListsUnitSuite) TestSharePointInfo() { + tests := []struct { + name string + listAndDeets func() (models.Listable, *details.SharePointInfo) + }{ + { + name: "Empty List", + listAndDeets: func() (models.Listable, *details.SharePointInfo) { + i := &details.SharePointInfo{ItemType: details.SharePointList} + return models.NewList(), i + }, + }, { + name: "Only Name", + listAndDeets: func() (models.Listable, *details.SharePointInfo) { + aTitle := "Whole List" + listing := models.NewList() + listing.SetDisplayName(&aTitle) + i := &details.SharePointInfo{ + ItemType: details.SharePointList, + ItemName: aTitle, + } + + return listing, i + }, + }, + } + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + list, expected := test.listAndDeets() + info := listToSPInfo(list, 10) + assert.Equal(t, expected.ItemType, info.ItemType) + assert.Equal(t, expected.ItemName, info.ItemName) + assert.Equal(t, expected.WebURL, info.WebURL) + }) + } +} diff --git a/src/internal/m365/sharepoint/pageInfo.go b/src/internal/m365/sharepoint/pages.go similarity index 84% rename from src/internal/m365/sharepoint/pageInfo.go rename to src/internal/m365/sharepoint/pages.go index 8b5060bdd..c5e0bb633 100644 --- a/src/internal/m365/sharepoint/pageInfo.go +++ b/src/internal/m365/sharepoint/pages.go @@ -8,10 +8,10 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" ) -// sharePointPageInfo propagates metadata from the SharePoint Page data type +// pageToSPInfo propagates metadata from the SharePoint Page data type // into searchable content. // Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta -func sharePointPageInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo { +func pageToSPInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo { var ( name, prefix, webURL string created, modified time.Time diff --git a/src/internal/m365/sharepoint/pageInfo_test.go b/src/internal/m365/sharepoint/pages_test.go similarity index 76% rename from src/internal/m365/sharepoint/pageInfo_test.go rename to src/internal/m365/sharepoint/pages_test.go index 7490b117f..d89b0d921 100644 --- a/src/internal/m365/sharepoint/pageInfo_test.go +++ b/src/internal/m365/sharepoint/pages_test.go @@ -1,13 +1,25 @@ package sharepoint import ( + "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/m365/graph/betasdk/models" + "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" ) -func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { +type PagesUnitSuite struct { + tester.Suite +} + +func TestPagesUnitSuite(t *testing.T) { + suite.Run(t, &PagesUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *PagesUnitSuite) TestSharePointInfo_Pages() { tests := []struct { name string pageAndDeets func() (models.SitePageable, *details.SharePointInfo) @@ -39,7 +51,7 @@ func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { t := suite.T() paged, expected := test.pageAndDeets() - info := sharePointPageInfo(paged, "", 0) + info := pageToSPInfo(paged, "", 0) assert.Equal(t, expected.ItemType, info.ItemType) assert.Equal(t, expected.ItemName, info.ItemName) assert.Equal(t, expected.WebURL, info.WebURL) diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index 7076d4de9..772515fd4 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -129,7 +129,7 @@ func restoreListItem( itemData data.Stream, siteID, destName string, ) (details.ItemInfo, error) { - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID())) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID())) defer end() ctx = clues.Add(ctx, "list_item_id", itemData.UUID()) @@ -190,7 +190,7 @@ func restoreListItem( } } - dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray))) + dii.SharePoint = listToSPInfo(restoredList, int64(len(byteArray))) return dii, nil } @@ -203,7 +203,7 @@ func RestoreListCollection( deets *details.Builder, errs *fault.Bus, ) (support.CollectionMetrics, error) { - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath())) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath())) defer end() var ( @@ -214,7 +214,7 @@ func RestoreListCollection( el = errs.Local() ) - trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String()) + trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String()) for { if el.Failure() != nil { @@ -285,8 +285,8 @@ func RestorePageCollection( siteID = directory.ResourceOwner() ) - trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String()) - ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath())) + trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String()) + ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath())) defer end() diff --git a/src/internal/m365/graph_connector_test_helper.go b/src/internal/m365/stub/stub.go similarity index 71% rename from src/internal/m365/graph_connector_test_helper.go rename to src/internal/m365/stub/stub.go index f95d1781f..601e57722 100644 --- a/src/internal/m365/graph_connector_test_helper.go +++ b/src/internal/m365/stub/stub.go @@ -1,12 +1,14 @@ -package m365 +package stub import ( "bytes" - "context" "io" + "golang.org/x/exp/maps" + "github.com/alcionai/corso/src/internal/data" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/pkg/control" @@ -33,9 +35,9 @@ type ItemInfo struct { // be the same before and after restoring the item in M365 and may not be // the M365 ID. When restoring items out of place, the item is assigned a // new ID making it unsuitable for a lookup key. - lookupKey string - name string - data []byte + LookupKey string + Name string + Data []byte } type ConfigInfo struct { @@ -47,19 +49,113 @@ type ConfigInfo struct { RestoreCfg control.RestoreConfig } -func mustToDataLayerPath( - service path.ServiceType, - tenant, resourceOwner string, - category path.CategoryType, - elements []string, - isItem bool, -) (path.Path, error) { - res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...) - if err != nil { - return nil, err +func GetCollectionsAndExpected( + config ConfigInfo, + testCollections []ColInfo, + backupVersion int, +) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { + var ( + collections []data.RestoreCollection + expectedData = map[string]map[string][]byte{} + totalItems = 0 + totalKopiaItems = 0 + ) + + for _, owner := range config.ResourceOwners { + numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo( + config.Service, + config.Tenant, + owner, + config.RestoreCfg, + testCollections, + backupVersion, + ) + if err != nil { + return totalItems, totalKopiaItems, collections, expectedData, err + } + + collections = append(collections, ownerCollections...) + totalItems += numItems + totalKopiaItems += kopiaItems + + maps.Copy(expectedData, userExpectedData) } - return res, err + return totalItems, totalKopiaItems, collections, expectedData, nil +} + +func CollectionsForInfo( + service path.ServiceType, + tenant, user string, + restoreCfg control.RestoreConfig, + allInfo []ColInfo, + backupVersion int, +) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { + var ( + collections = make([]data.RestoreCollection, 0, len(allInfo)) + expectedData = make(map[string]map[string][]byte, len(allInfo)) + totalItems = 0 + kopiaEntries = 0 + ) + + for _, info := range allInfo { + pth, err := path.Build( + tenant, + user, + service, + info.Category, + false, + info.PathElements...) + if err != nil { + return totalItems, kopiaEntries, collections, expectedData, err + } + + mc := exchMock.NewCollection(pth, pth, len(info.Items)) + + baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth) + if err != nil { + return totalItems, kopiaEntries, collections, expectedData, err + } + + baseExpected := expectedData[baseDestPath.String()] + if baseExpected == nil { + expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items)) + baseExpected = expectedData[baseDestPath.String()] + } + + for i := 0; i < len(info.Items); i++ { + mc.Names[i] = info.Items[i].Name + mc.Data[i] = info.Items[i].Data + + baseExpected[info.Items[i].LookupKey] = info.Items[i].Data + + // We do not count metadata files against item count + if backupVersion > 0 && + (service == path.OneDriveService || service == path.SharePointService) && + metadata.HasMetaSuffix(info.Items[i].Name) { + continue + } + + totalItems++ + } + + c := mock.RestoreCollection{ + Collection: mc, + AuxItems: map[string]data.Stream{}, + } + + for _, aux := range info.AuxItems { + c.AuxItems[aux.Name] = &exchMock.Data{ + ID: aux.Name, + Reader: io.NopCloser(bytes.NewReader(aux.Data)), + } + } + + collections = append(collections, c) + kopiaEntries += len(info.Items) + } + + return totalItems, kopiaEntries, collections, expectedData, nil } // backupOutputPathFromRestore returns a path.Path denoting the location in @@ -86,102 +182,11 @@ func backupOutputPathFromRestore( base = append(base, inputPath.Folders()...) } - return mustToDataLayerPath( - inputPath.Service(), + return path.Build( inputPath.Tenant(), inputPath.ResourceOwner(), + inputPath.Service(), inputPath.Category(), - base, false, - ) -} - -// TODO(ashmrtn): Make this an actual mock class that can be used in other -// packages. -type mockRestoreCollection struct { - data.Collection - auxItems map[string]data.Stream -} - -func (rc mockRestoreCollection) FetchItemByName( - ctx context.Context, - name string, -) (data.Stream, error) { - res := rc.auxItems[name] - if res == nil { - return nil, data.ErrNotFound - } - - return res, nil -} - -func collectionsForInfo( - service path.ServiceType, - tenant, user string, - restoreCfg control.RestoreConfig, - allInfo []ColInfo, - backupVersion int, -) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) { - var ( - collections = make([]data.RestoreCollection, 0, len(allInfo)) - expectedData = make(map[string]map[string][]byte, len(allInfo)) - totalItems = 0 - kopiaEntries = 0 - ) - - for _, info := range allInfo { - pth, err := mustToDataLayerPath( - service, - tenant, - user, - info.Category, - info.PathElements, - false) - if err != nil { - return totalItems, kopiaEntries, collections, expectedData, err - } - - mc := exchMock.NewCollection(pth, pth, len(info.Items)) - - baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth) - if err != nil { - return totalItems, kopiaEntries, collections, expectedData, err - } - - baseExpected := expectedData[baseDestPath.String()] - if baseExpected == nil { - expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items)) - baseExpected = expectedData[baseDestPath.String()] - } - - for i := 0; i < len(info.Items); i++ { - mc.Names[i] = info.Items[i].name - mc.Data[i] = info.Items[i].data - - baseExpected[info.Items[i].lookupKey] = info.Items[i].data - - // We do not count metadata files against item count - if backupVersion > 0 && - (service == path.OneDriveService || service == path.SharePointService) && - metadata.HasMetaSuffix(info.Items[i].name) { - continue - } - - totalItems++ - } - - c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}} - - for _, aux := range info.AuxItems { - c.auxItems[aux.name] = &exchMock.Data{ - ID: aux.name, - Reader: io.NopCloser(bytes.NewReader(aux.data)), - } - } - - collections = append(collections, c) - kopiaEntries += len(info.Items) - } - - return totalItems, kopiaEntries, collections, expectedData, nil + base...) }