From 48aae5d485494581c5939b7b46f259875be8a31c Mon Sep 17 00:00:00 2001 From: ryanfkeepers Date: Fri, 11 Aug 2023 16:16:40 -0600 Subject: [PATCH] all non-resourcer compliance Updates all path package uses that does not involve a Resourcer. Third to last step to wrapping up compliance. Second step is to update the Resourcer interface to comply with ServiceResources. Last step is to clean up any lingering linters, bugs, tests, and other things needed to make the build green. --- src/internal/kopia/upload.go | 2 +- src/internal/m365/backup_test.go | 18 +-- .../m365/collection/site/collection.go | 14 ++- src/internal/m365/collection/site/restore.go | 18 ++- src/internal/m365/controller_test.go | 18 +-- src/internal/m365/graph/metadata/metadata.go | 25 +++- .../m365/graph/metadata/metadata_test.go | 118 ++++++++++++++---- src/internal/m365/helper_test.go | 26 ++-- .../m365/service/exchange/backup_test.go | 41 ++++-- src/internal/m365/service/groups/restore.go | 4 +- .../m365/service/sharepoint/restore.go | 4 +- src/internal/m365/stub/stub.go | 23 ++-- src/internal/operations/backup.go | 5 +- src/internal/operations/backup_test.go | 54 ++++---- src/internal/streamstore/streamstore.go | 14 ++- src/pkg/path/resource_path.go | 2 +- src/pkg/path/service_resource.go | 15 ++- src/pkg/path/service_resource_test.go | 2 +- 18 files changed, 284 insertions(+), 119 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 522d3fad5..2d815594f 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -515,7 +515,7 @@ func streamBaseEntries( // TODO(ashmrtn): We may eventually want to make this a function that is // passed in so that we can more easily switch it between different external // service provider implementations. - if !metadata.IsMetadataFile(itemPath) { + if !metadata.IsMetadataFilePath(itemPath) { // All items have item info in the base backup. However, we need to make // sure we have enough metadata to find those entries. To do that we add // the item to progress and having progress aggregate everything for diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 3e35b0030..c00167a69 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -403,17 +403,19 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { // No excludes yet as this isn't an incremental backup. assert.True(t, excludes.Empty()) + // assume the last service in the path is sharepoint. + srs := cols[0].FullPath().ServiceResources() + service := srs[len(srs)-1].Service + t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String()) - assert.Equal( - t, - path.SharePointMetadataService.String(), - cols[0].FullPath().Service().String()) + assert.Equal(t, path.SharePointMetadataService, service) + + // assume the last service in the path is sharepoint. + srs = cols[1].FullPath().ServiceResources() + service = srs[len(srs)-1].Service t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String()) - assert.Equal( - t, - path.SharePointService.String(), - cols[1].FullPath().Service().String()) + assert.Equal(t, path.SharePointService, service) } func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { diff --git a/src/internal/m365/collection/site/collection.go b/src/internal/m365/collection/site/collection.go index 6d115ca3b..d4d5f82e5 100644 --- a/src/internal/m365/collection/site/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -218,12 +218,16 @@ func (sc *Collection) retrieveLists( var ( metrics support.CollectionMetrics el = errs.Local() + // todo: pass in the resourceOwner as an idname.Provider + srs = sc.fullPath.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) lists, err := loadSiteLists( ctx, sc.client.Stable, - sc.fullPath.ResourceOwner(), + protectedResource, sc.jobs, errs) if err != nil { @@ -279,6 +283,10 @@ func (sc *Collection) retrievePages( var ( metrics support.CollectionMetrics el = errs.Local() + // todo: pass in the resourceOwner as an idname.Provider + srs = sc.fullPath.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) betaService := sc.betaService @@ -286,14 +294,14 @@ func (sc *Collection) retrievePages( return metrics, clues.New("beta service required").WithClues(ctx) } - parent, err := as.GetByID(ctx, sc.fullPath.ResourceOwner()) + parent, err := as.GetByID(ctx, protectedResource) if err != nil { return metrics, err } root := ptr.Val(parent.GetWebUrl()) - pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs) + pages, err := betaAPI.GetSitePages(ctx, betaService, protectedResource, sc.jobs, errs) if err != nil { return metrics, err } diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index 875ac5115..d89c949f5 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -69,7 +69,9 @@ func ConsumeRestoreCollections( ictx = clues.Add(ctx, "category", category, "restore_location", clues.Hide(rcc.RestoreConfig.Location), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owners", clues.Hide( + path.ServiceResourcesToResources( + dc.FullPath().ServiceResources())), "full_path", dc.FullPath()) ) @@ -219,9 +221,12 @@ func RestoreListCollection( var ( metrics = support.CollectionMetrics{} directory = dc.FullPath() - siteID = directory.ResourceOwner() items = dc.Items(ctx, errs) el = errs.Local() + // todo: pass in the resourceOwner as an idname.Provider + srs = directory.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String()) @@ -245,7 +250,7 @@ func RestoreListCollection( ctx, service, itemData, - siteID, + protectedResource, restoreContainerName) if err != nil { el.AddRecoverable(ctx, err) @@ -292,7 +297,10 @@ func RestorePageCollection( var ( metrics = support.CollectionMetrics{} directory = dc.FullPath() - siteID = directory.ResourceOwner() + // todo: pass in the resourceOwner as an idname.Provider + srs = directory.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String()) @@ -325,7 +333,7 @@ func RestorePageCollection( ctx, service, itemData, - siteID, + protectedResource, restoreContainerName) if err != nil { el.AddRecoverable(ctx, err) diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 73bb65b93..9730f6cb3 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -1249,11 +1249,11 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { table := []struct { - name string - resourceCat resource.Category - selectorFunc func(t *testing.T) selectors.Selector - service path.ServiceType - categories []string + name string + resourceCat resource.Category + selectorFunc func(t *testing.T) selectors.Selector + metadataServices []path.ServiceType + categories []string }{ { name: "Exchange", @@ -1267,7 +1267,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { return sel.Selector }, - service: path.ExchangeService, + metadataServices: []path.ServiceType{path.ExchangeMetadataService}, categories: []string{ path.EmailCategory.String(), path.ContactsCategory.String(), @@ -1283,7 +1283,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { return sel.Selector }, - service: path.OneDriveService, + metadataServices: []path.ServiceType{path.OneDriveMetadataService}, categories: []string{ path.FilesCategory.String(), }, @@ -1302,7 +1302,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { return sel.Selector }, - service: path.SharePointService, + metadataServices: []path.ServiceType{path.SharePointMetadataService}, categories: []string{ path.LibrariesCategory.String(), // not yet in use @@ -1365,7 +1365,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { // Ignore metadata collections. fullPath := col.FullPath() - if fullPath.Service() != test.service { + if path.ServiceResourcesMatchServices(fullPath.ServiceResources(), test.metadataServices) { continue } diff --git a/src/internal/m365/graph/metadata/metadata.go b/src/internal/m365/graph/metadata/metadata.go index d213cd481..61e01ff23 100644 --- a/src/internal/m365/graph/metadata/metadata.go +++ b/src/internal/m365/graph/metadata/metadata.go @@ -5,13 +5,30 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -func IsMetadataFile(p path.Path) bool { - switch p.Service() { +// IsMetadataFilePath checks whether the LAST service in the path +// supports metadata file types and, if so, whether the item has +// a meta suffix. +func IsMetadataFilePath(p path.Path) bool { + return IsMetadataFile( + p.ServiceResources(), + p.Category(), + p.Item()) +} + +// IsMetadataFile accepts the ServiceResources, cat, and Item values from +// a path (or equivalent representation) and returns true if the item +// is a Metadata entry. +func IsMetadataFile( + srs []path.ServiceResource, + cat path.CategoryType, + itemID string, +) bool { + switch srs[len(srs)-1].Service { case path.OneDriveService: - return metadata.HasMetaSuffix(p.Item()) + return metadata.HasMetaSuffix(itemID) case path.SharePointService: - return p.Category() == path.LibrariesCategory && metadata.HasMetaSuffix(p.Item()) + return cat == path.LibrariesCategory && metadata.HasMetaSuffix(itemID) default: return false diff --git a/src/internal/m365/graph/metadata/metadata_test.go b/src/internal/m365/graph/metadata/metadata_test.go index 256f558f8..c62d7935e 100644 --- a/src/internal/m365/graph/metadata/metadata_test.go +++ b/src/internal/m365/graph/metadata/metadata_test.go @@ -1,7 +1,7 @@ package metadata_test import ( - "fmt" + "strings" "testing" "github.com/alcionai/clues" @@ -18,7 +18,7 @@ import ( type boolfAssertionFunc func(assert.TestingT, bool, string, ...any) bool type testCase struct { - service path.ServiceType + srs []path.ServiceResource category path.CategoryType expected boolfAssertionFunc } @@ -39,40 +39,89 @@ var ( cases = []testCase{ { - service: path.ExchangeService, + srs: []path.ServiceResource{{ + Service: path.ExchangeService, + ProtectedResource: user, + }}, category: path.EmailCategory, expected: assert.Falsef, }, { - service: path.ExchangeService, + srs: []path.ServiceResource{{ + Service: path.ExchangeService, + ProtectedResource: user, + }}, category: path.ContactsCategory, expected: assert.Falsef, }, { - service: path.ExchangeService, + srs: []path.ServiceResource{{ + Service: path.ExchangeService, + ProtectedResource: user, + }}, category: path.EventsCategory, expected: assert.Falsef, }, { - service: path.OneDriveService, + srs: []path.ServiceResource{{ + Service: path.OneDriveService, + ProtectedResource: user, + }}, category: path.FilesCategory, expected: assert.Truef, }, { - service: path.SharePointService, + srs: []path.ServiceResource{{ + Service: path.SharePointService, + ProtectedResource: user, + }}, category: path.LibrariesCategory, expected: assert.Truef, }, { - service: path.SharePointService, + srs: []path.ServiceResource{{ + Service: path.SharePointService, + ProtectedResource: user, + }}, category: path.ListsCategory, expected: assert.Falsef, }, { - service: path.SharePointService, + srs: []path.ServiceResource{{ + Service: path.SharePointService, + ProtectedResource: user, + }}, category: path.PagesCategory, expected: assert.Falsef, }, + { + srs: []path.ServiceResource{ + { + Service: path.OneDriveService, + ProtectedResource: user, + }, + { + Service: path.ExchangeService, + ProtectedResource: user, + }, + }, + category: path.EventsCategory, + expected: assert.Falsef, + }, + { + srs: []path.ServiceResource{ + { + Service: path.ExchangeService, + ProtectedResource: user, + }, + { + Service: path.OneDriveService, + ProtectedResource: user, + }, + }, + category: path.FilesCategory, + expected: assert.Truef, + }, } ) @@ -87,21 +136,26 @@ func TestMetadataUnitSuite(t *testing.T) { func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() { for _, test := range cases { for _, ext := range metaSuffixes { - suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { + name := []string{} + + for _, sr := range test.srs { + name = append(name, sr.Service.String()) + } + + name = append(name, test.category.String(), ext) + + suite.Run(strings.Join(name, " "), func() { t := suite.T() p, err := path.Build( tenant, - []path.ServiceResource{{ - Service: test.service, - ProtectedResource: user, - }}, + test.srs, test.category, true, "file"+ext) require.NoError(t, err, clues.ToCore(err)) - test.expected(t, metadata.IsMetadataFile(p), "extension %s", ext) + test.expected(t, metadata.IsMetadataFilePath(p), "extension %s", ext) }) } } @@ -110,21 +164,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() { func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_NotMetaSuffixes() { for _, test := range cases { for _, ext := range notMetaSuffixes { - suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { + name := []string{} + + for _, sr := range test.srs { + name = append(name, sr.Service.String()) + } + + name = append(name, test.category.String(), ext) + + suite.Run(strings.Join(name, " "), func() { t := suite.T() p, err := path.Build( tenant, - []path.ServiceResource{{ - Service: test.service, - ProtectedResource: user, - }}, + test.srs, test.category, true, "file"+ext) require.NoError(t, err, clues.ToCore(err)) - assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext) + assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext) }) } } @@ -135,21 +194,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Directories() { for _, test := range cases { for _, ext := range suffixes { - suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { + name := []string{} + + for _, sr := range test.srs { + name = append(name, sr.Service.String()) + } + + name = append(name, test.category.String(), ext) + + suite.Run(strings.Join(name, " "), func() { t := suite.T() p, err := path.Build( tenant, - []path.ServiceResource{{ - Service: test.service, - ProtectedResource: user, - }}, + test.srs, test.category, false, "file"+ext) require.NoError(t, err, clues.ToCore(err)) - assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext) + assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext) }) } } diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index fee6cc1f5..39d2f6606 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -865,7 +865,6 @@ func compareItem( t *testing.T, colPath path.Path, expected map[string][]byte, - service path.ServiceType, category path.CategoryType, item data.Stream, mci m365Stub.ConfigInfo, @@ -875,7 +874,11 @@ func compareItem( assert.NotZero(t, mt.ModTime()) } - switch service { + // assume the last service in the path is the data owner + srs := colPath.ServiceResources() + lastService := srs[len(srs)-1].Service + + switch lastService { case path.ExchangeService: switch category { case path.EmailCategory: @@ -900,7 +903,7 @@ func compareItem( return compareDriveItem(t, expected, item, mci, rootDir) default: - assert.FailNowf(t, "unexpected service: %s", service.String()) + assert.FailNowf(t, "unexpected service: %s", lastService.String()) } return true @@ -929,9 +932,12 @@ func checkHasCollections( fp := g.FullPath() loc := g.(data.LocationPather).LocationPath() + // take the last service, since it should be the one owning data + srs := fp.ServiceResources() + service := srs[len(srs)-1].Service - if fp.Service() == path.OneDriveService || - (fp.Service() == path.SharePointService && fp.Category() == path.LibrariesCategory) { + if service == path.OneDriveService || + (service == path.SharePointService && fp.Category() == path.LibrariesCategory) { dp, err := path.ToDrivePath(fp) if !assert.NoError(t, err, clues.ToCore(err)) { continue @@ -971,11 +977,12 @@ func checkCollections( for _, returned := range got { var ( hasItems bool - service = returned.FullPath().Service() category = returned.FullPath().Category() expectedColData = expected[returned.FullPath().String()] folders = returned.FullPath().Elements() rootDir = folders[len(folders)-1] == mci.RestoreCfg.Location + srs = returned.FullPath().ServiceResources() + lastService = srs[len(srs)-1].Service ) // Need to iterate through all items even if we don't expect to find a match @@ -987,9 +994,9 @@ func checkCollections( // is for actual pull items. // TODO(ashmrtn): Should probably eventually check some data in metadata // collections. - if service == path.ExchangeMetadataService || - service == path.OneDriveMetadataService || - service == path.SharePointMetadataService { + if lastService == path.ExchangeMetadataService || + lastService == path.OneDriveMetadataService || + lastService == path.SharePointMetadataService { skipped++ continue } @@ -1005,7 +1012,6 @@ func checkCollections( t, returned.FullPath(), expectedColData, - service, category, item, mci, diff --git a/src/internal/m365/service/exchange/backup_test.go b/src/internal/m365/service/exchange/backup_test.go index acee5119e..e57cf2ce9 100644 --- a/src/internal/m365/service/exchange/backup_test.go +++ b/src/internal/m365/service/exchange/backup_test.go @@ -491,7 +491,9 @@ func (suite *BackupIntgSuite) TestMailFetch() { require.NoError(t, err, clues.ToCore(err)) for _, c := range collections { - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { continue } @@ -577,7 +579,9 @@ func (suite *BackupIntgSuite) TestDelta() { var metadata data.BackupCollection for _, coll := range collections { - if coll.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + coll.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadata = coll } } @@ -611,7 +615,9 @@ func (suite *BackupIntgSuite) TestDelta() { // Delta usage is commented out at the moment, anyway. So this is currently // a sanity check that the minimum behavior won't break. for _, coll := range collections { - if coll.FullPath().Service() != path.ExchangeMetadataService { + if !path.ServiceResourcesMatchServices( + coll.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { ec, ok := coll.(*Collection) require.True(t, ok, "collection is *Collection") assert.NotNil(t, ec) @@ -666,7 +672,9 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() { ctx, flush := tester.NewContext(t) defer flush() - isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService + isMetadata := path.ServiceResourcesMatchServices( + edc.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) streamChannel := edc.Items(ctx, fault.New(true)) // Verify that each message can be restored @@ -744,7 +752,9 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() { require.GreaterOrEqual(t, 2, len(edcs), "expected 1 <= num collections <= 2") for _, edc := range edcs { - isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService + isMetadata := path.ServiceResourcesMatchServices( + edc.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) count := 0 for stream := range edc.Items(ctx, fault.New(true)) { @@ -874,7 +884,10 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() { for _, edc := range collections { var isMetadata bool - if edc.FullPath().Service() != path.ExchangeMetadataService { + // FIXME: this doesn't seem right, it's saying "if not metadata, isMetadata = true" + if !path.ServiceResourcesMatchServices( + edc.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { isMetadata = true assert.Equal(t, test.expected, edc.FullPath().Folder(false)) } else { @@ -1140,7 +1153,9 @@ func (suite *CollectionPopulationSuite) TestPopulateCollections() { deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0 for _, c := range collections { - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ continue } @@ -1491,7 +1506,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D continue } - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c) continue @@ -1650,7 +1667,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_r deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0 for _, c := range collections { - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ continue } @@ -2082,7 +2101,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i require.NotNil(t, p) - if p.Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ continue } diff --git a/src/internal/m365/service/groups/restore.go b/src/internal/m365/service/groups/restore.go index e36b3d7df..5512df4c6 100644 --- a/src/internal/m365/service/groups/restore.go +++ b/src/internal/m365/service/groups/restore.go @@ -57,7 +57,9 @@ func ConsumeRestoreCollections( ictx = clues.Add(ctx, "category", category, "restore_location", clues.Hide(rcc.RestoreConfig.Location), - "protected_resource", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owners", clues.Hide( + path.ServiceResourcesToResources( + dc.FullPath().ServiceResources())), "full_path", dc.FullPath()) ) diff --git a/src/internal/m365/service/sharepoint/restore.go b/src/internal/m365/service/sharepoint/restore.go index 35e1c67cd..12aaad60e 100644 --- a/src/internal/m365/service/sharepoint/restore.go +++ b/src/internal/m365/service/sharepoint/restore.go @@ -61,7 +61,9 @@ func ConsumeRestoreCollections( ictx = clues.Add(ctx, "category", category, "restore_location", clues.Hide(rcc.RestoreConfig.Location), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owners", clues.Hide( + path.ServiceResourcesToResources( + dc.FullPath().ServiceResources())), "full_path", dc.FullPath()) ) diff --git a/src/internal/m365/stub/stub.go b/src/internal/m365/stub/stub.go index 133fc5ff1..c0d4459c1 100644 --- a/src/internal/m365/stub/stub.go +++ b/src/internal/m365/stub/stub.go @@ -7,7 +7,7 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + "github.com/alcionai/corso/src/internal/m365/graph/metadata" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" @@ -63,9 +63,11 @@ func GetCollectionsAndExpected( for _, owner := range config.ResourceOwners { numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo( - config.Service, config.Tenant, - owner, + []path.ServiceResource{{ + Service: config.Service, + ProtectedResource: owner, + }}, config.RestoreCfg, testCollections, backupVersion) @@ -128,9 +130,7 @@ func CollectionsForInfo( baseExpected[info.Items[i].LookupKey] = info.Items[i].Data // We do not count metadata files against item count - if backupVersion > 0 && - (service == path.OneDriveService || service == path.SharePointService) && - metadata.HasMetaSuffix(info.Items[i].Name) { + if backupVersion > 0 && metadata.IsMetadataFile(srs, info.Category, info.Items[i].Name) { continue } @@ -165,9 +165,13 @@ func backupOutputPathFromRestore( inputPath path.Path, ) (path.Path, error) { base := []string{restoreCfg.Location} + srs := inputPath.ServiceResources() + // only the last service is checked, because that should be the service + // whose data is stored.. + lastService := srs[len(srs)-1].Service // OneDrive has leading information like the drive ID. - if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService { + if lastService == path.OneDriveService || lastService == path.SharePointService { folders := inputPath.Folders() base = append(append([]string{}, folders[:3]...), restoreCfg.Location) @@ -176,14 +180,13 @@ func backupOutputPathFromRestore( } } - if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory { + if lastService == path.ExchangeService && inputPath.Category() == path.EmailCategory { base = append(base, inputPath.Folders()...) } return path.Build( inputPath.Tenant(), - inputPath.ResourceOwner(), - inputPath.Service(), + inputPath.ServiceResources(), inputPath.Category(), false, base...) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index dbe7d0d5c..2dead366f 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -566,7 +566,10 @@ func getNewPathRefs( // TODO(ashmrtn): In the future we can remove this first check as we'll be // able to assume we always have the location in the previous entry. We'll end // up doing some extra parsing, but it will simplify this code. - if repoRef.Service() == path.ExchangeService { + // + // Currently safe to check only the 0th SR, since exchange had no subservices + // at the time of the locations addition. + if repoRef.ServiceResources()[0].Service == path.ExchangeService { newPath, newLoc, err := dataFromBackup.GetNewPathRefs( repoRef.ToBuilder(), entry.Modified(), diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index d5e0ee3f3..5538c4ce5 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -266,6 +266,7 @@ func makePath(t *testing.T, elements []string, isItem bool) path.Path { return p } +// FIXME: out of date, does not contain sharepoint support func makeDetailsEntry( t *testing.T, p path.Path, @@ -290,7 +291,10 @@ func makeDetailsEntry( Updated: updated, } - switch p.Service() { + srs := p.ServiceResources() + lastService := srs[len(srs)-1].Service + + switch lastService { case path.ExchangeService: if p.Category() != path.EmailCategory { assert.FailNowf( @@ -321,7 +325,7 @@ func makeDetailsEntry( assert.FailNowf( t, "service %s not supported in helper function", - p.Service().String()) + lastService.String()) } return res @@ -529,6 +533,23 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections fault.New(true)) } +// makeElements allows creation of repoRefs that wouldn't +// pass paths package validators. +func makeElements( + tenant string, + srs []path.ServiceResource, + cat path.CategoryType, + suffix ...string, +) path.Elements { + elems := append( + path.Elements{tenant}, + path.ServiceResourcesToElements(srs)...) + elems = append(elems, cat.String()) + elems = append(elems, suffix...) + + return elems +} + func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() { var ( tenant = "a-tenant" @@ -709,17 +730,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems DetailsModel: details.DetailsModel{ Entries: []details.Entry{ { - RepoRef: stdpath.Join( - append( - []string{ - itemPath1.Tenant(), - itemPath1.Service().String(), - itemPath1.ResourceOwner(), - path.UnknownCategory.String(), - }, - itemPath1.Folders()..., - )..., - ), + RepoRef: stdpath.Join(makeElements( + itemPath1.Tenant(), + itemPath1.ServiceResources(), + path.UnknownCategory, + itemPath1.Folders()...)...), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -740,16 +755,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems res := newMockDetailsMergeInfoer() p := makePath( suite.T(), - []string{ + makeElements( itemPath1.Tenant(), - path.OneDriveService.String(), - itemPath1.ResourceOwner(), - path.FilesCategory.String(), + itemPath1.ServiceResources(), + path.FilesCategory, "personal", - "item1", - }, - true, - ) + "item1"), + true) res.add(itemPath1, p, nil) diff --git a/src/internal/streamstore/streamstore.go b/src/internal/streamstore/streamstore.go index 6f5918c81..49d9f16e9 100644 --- a/src/internal/streamstore/streamstore.go +++ b/src/internal/streamstore/streamstore.go @@ -198,8 +198,13 @@ func collect( service path.ServiceType, col Collectable, ) (data.BackupCollection, error) { + srs := []path.ServiceResource{{ + Service: service, + ProtectedResource: col.purpose, + }} + // construct the path of the container - p, err := path.Builder{}.ToStreamStorePath(tenantID, col.purpose, service, false) + p, err := path.Builder{}.ToStreamStorePath(tenantID, srs, false) if err != nil { return nil, clues.Stack(err).WithClues(ctx) } @@ -257,10 +262,15 @@ func read( rer inject.RestoreProducer, errs *fault.Bus, ) error { + srs := []path.ServiceResource{{ + Service: service, + ProtectedResource: col.purpose, + }} + // construct the path of the container p, err := path.Builder{}. Append(col.itemName). - ToStreamStorePath(tenantID, col.purpose, service, true) + ToStreamStorePath(tenantID, srs, true) if err != nil { return clues.Stack(err).WithClues(ctx) } diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index a8cb6222a..1bd380286 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -31,7 +31,7 @@ func newDataLayerResourcePath( cat CategoryType, isItem bool, ) dataLayerResourcePath { - pfx := append([]string{tenant}, serviceResourcesToElements(srs)...) + pfx := append([]string{tenant}, ServiceResourcesToElements(srs)...) pfx = append(pfx, cat.String()) return dataLayerResourcePath{ diff --git a/src/pkg/path/service_resource.go b/src/pkg/path/service_resource.go index 77317d029..037011a24 100644 --- a/src/pkg/path/service_resource.go +++ b/src/pkg/path/service_resource.go @@ -2,6 +2,7 @@ package path import ( "github.com/alcionai/clues" + "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/tform" @@ -84,11 +85,13 @@ func ServiceResourcesToResources(srs []ServiceResource) []string { return prs } -// --------------------------------------------------------------------------- -// Unexported Helpers -// --------------------------------------------------------------------------- +func ServiceResourcesMatchServices(srs []ServiceResource, sts []ServiceType) bool { + return slices.EqualFunc(srs, sts, func(sr ServiceResource, st ServiceType) bool { + return sr.Service == st + }) +} -func serviceResourcesToElements(srs []ServiceResource) Elements { +func ServiceResourcesToElements(srs []ServiceResource) Elements { es := make(Elements, 0, len(srs)*2) for _, tuple := range srs { @@ -99,6 +102,10 @@ func serviceResourcesToElements(srs []ServiceResource) Elements { return es } +// --------------------------------------------------------------------------- +// Unexported Helpers +// --------------------------------------------------------------------------- + // elementsToServiceResources turns as many pairs of elems as possible // into ServiceResource tuples. Elems must begin with a service, but // may contain more entries than there are service/resource pairs. diff --git a/src/pkg/path/service_resource_test.go b/src/pkg/path/service_resource_test.go index 97139c49a..6eab81350 100644 --- a/src/pkg/path/service_resource_test.go +++ b/src/pkg/path/service_resource_test.go @@ -157,7 +157,7 @@ func (suite *ServiceResourceUnitSuite) TestServiceResourceToElements() { suite.Run(test.name, func() { t := suite.T() - result := serviceResourcesToElements(test.srs) + result := ServiceResourcesToElements(test.srs) // not ElementsMatch, order matters assert.Equal(t, test.expect, result)