diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 522d3fad5..2d815594f 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -515,7 +515,7 @@ func streamBaseEntries( // TODO(ashmrtn): We may eventually want to make this a function that is // passed in so that we can more easily switch it between different external // service provider implementations. - if !metadata.IsMetadataFile(itemPath) { + if !metadata.IsMetadataFilePath(itemPath) { // All items have item info in the base backup. However, we need to make // sure we have enough metadata to find those entries. To do that we add // the item to progress and having progress aggregate everything for diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 3e35b0030..c00167a69 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -403,17 +403,19 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { // No excludes yet as this isn't an incremental backup. assert.True(t, excludes.Empty()) + // assume the last service in the path is sharepoint. + srs := cols[0].FullPath().ServiceResources() + service := srs[len(srs)-1].Service + t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String()) - assert.Equal( - t, - path.SharePointMetadataService.String(), - cols[0].FullPath().Service().String()) + assert.Equal(t, path.SharePointMetadataService, service) + + // assume the last service in the path is sharepoint. + srs = cols[1].FullPath().ServiceResources() + service = srs[len(srs)-1].Service t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String()) - assert.Equal( - t, - path.SharePointService.String(), - cols[1].FullPath().Service().String()) + assert.Equal(t, path.SharePointService, service) } func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { diff --git a/src/internal/m365/collection/site/collection.go b/src/internal/m365/collection/site/collection.go index 6d115ca3b..d4d5f82e5 100644 --- a/src/internal/m365/collection/site/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -218,12 +218,16 @@ func (sc *Collection) retrieveLists( var ( metrics support.CollectionMetrics el = errs.Local() + // todo: pass in the resourceOwner as an idname.Provider + srs = sc.fullPath.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) lists, err := loadSiteLists( ctx, sc.client.Stable, - sc.fullPath.ResourceOwner(), + protectedResource, sc.jobs, errs) if err != nil { @@ -279,6 +283,10 @@ func (sc *Collection) retrievePages( var ( metrics support.CollectionMetrics el = errs.Local() + // todo: pass in the resourceOwner as an idname.Provider + srs = sc.fullPath.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) betaService := sc.betaService @@ -286,14 +294,14 @@ func (sc *Collection) retrievePages( return metrics, clues.New("beta service required").WithClues(ctx) } - parent, err := as.GetByID(ctx, sc.fullPath.ResourceOwner()) + parent, err := as.GetByID(ctx, protectedResource) if err != nil { return metrics, err } root := ptr.Val(parent.GetWebUrl()) - pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs) + pages, err := betaAPI.GetSitePages(ctx, betaService, protectedResource, sc.jobs, errs) if err != nil { return metrics, err } diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index 875ac5115..d89c949f5 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -69,7 +69,9 @@ func ConsumeRestoreCollections( ictx = clues.Add(ctx, "category", category, "restore_location", clues.Hide(rcc.RestoreConfig.Location), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owners", clues.Hide( + path.ServiceResourcesToResources( + dc.FullPath().ServiceResources())), "full_path", dc.FullPath()) ) @@ -219,9 +221,12 @@ func RestoreListCollection( var ( metrics = support.CollectionMetrics{} directory = dc.FullPath() - siteID = directory.ResourceOwner() items = dc.Items(ctx, errs) el = errs.Local() + // todo: pass in the resourceOwner as an idname.Provider + srs = directory.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String()) @@ -245,7 +250,7 @@ func RestoreListCollection( ctx, service, itemData, - siteID, + protectedResource, restoreContainerName) if err != nil { el.AddRecoverable(ctx, err) @@ -292,7 +297,10 @@ func RestorePageCollection( var ( metrics = support.CollectionMetrics{} directory = dc.FullPath() - siteID = directory.ResourceOwner() + // todo: pass in the resourceOwner as an idname.Provider + srs = directory.ServiceResources() + // take the last resource in srs, since that should be the data owner + protectedResource = srs[len(srs)-1].ProtectedResource ) trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String()) @@ -325,7 +333,7 @@ func RestorePageCollection( ctx, service, itemData, - siteID, + protectedResource, restoreContainerName) if err != nil { el.AddRecoverable(ctx, err) diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 73bb65b93..9730f6cb3 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -1249,11 +1249,11 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { table := []struct { - name string - resourceCat resource.Category - selectorFunc func(t *testing.T) selectors.Selector - service path.ServiceType - categories []string + name string + resourceCat resource.Category + selectorFunc func(t *testing.T) selectors.Selector + metadataServices []path.ServiceType + categories []string }{ { name: "Exchange", @@ -1267,7 +1267,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { return sel.Selector }, - service: path.ExchangeService, + metadataServices: []path.ServiceType{path.ExchangeMetadataService}, categories: []string{ path.EmailCategory.String(), path.ContactsCategory.String(), @@ -1283,7 +1283,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { return sel.Selector }, - service: path.OneDriveService, + metadataServices: []path.ServiceType{path.OneDriveMetadataService}, categories: []string{ path.FilesCategory.String(), }, @@ -1302,7 +1302,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { return sel.Selector }, - service: path.SharePointService, + metadataServices: []path.ServiceType{path.SharePointMetadataService}, categories: []string{ path.LibrariesCategory.String(), // not yet in use @@ -1365,7 +1365,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { // Ignore metadata collections. fullPath := col.FullPath() - if fullPath.Service() != test.service { + if path.ServiceResourcesMatchServices(fullPath.ServiceResources(), test.metadataServices) { continue } diff --git a/src/internal/m365/graph/metadata/metadata.go b/src/internal/m365/graph/metadata/metadata.go index d213cd481..61e01ff23 100644 --- a/src/internal/m365/graph/metadata/metadata.go +++ b/src/internal/m365/graph/metadata/metadata.go @@ -5,13 +5,30 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -func IsMetadataFile(p path.Path) bool { - switch p.Service() { +// IsMetadataFilePath checks whether the LAST service in the path +// supports metadata file types and, if so, whether the item has +// a meta suffix. +func IsMetadataFilePath(p path.Path) bool { + return IsMetadataFile( + p.ServiceResources(), + p.Category(), + p.Item()) +} + +// IsMetadataFile accepts the ServiceResources, cat, and Item values from +// a path (or equivalent representation) and returns true if the item +// is a Metadata entry. +func IsMetadataFile( + srs []path.ServiceResource, + cat path.CategoryType, + itemID string, +) bool { + switch srs[len(srs)-1].Service { case path.OneDriveService: - return metadata.HasMetaSuffix(p.Item()) + return metadata.HasMetaSuffix(itemID) case path.SharePointService: - return p.Category() == path.LibrariesCategory && metadata.HasMetaSuffix(p.Item()) + return cat == path.LibrariesCategory && metadata.HasMetaSuffix(itemID) default: return false diff --git a/src/internal/m365/graph/metadata/metadata_test.go b/src/internal/m365/graph/metadata/metadata_test.go index 256f558f8..c62d7935e 100644 --- a/src/internal/m365/graph/metadata/metadata_test.go +++ b/src/internal/m365/graph/metadata/metadata_test.go @@ -1,7 +1,7 @@ package metadata_test import ( - "fmt" + "strings" "testing" "github.com/alcionai/clues" @@ -18,7 +18,7 @@ import ( type boolfAssertionFunc func(assert.TestingT, bool, string, ...any) bool type testCase struct { - service path.ServiceType + srs []path.ServiceResource category path.CategoryType expected boolfAssertionFunc } @@ -39,40 +39,89 @@ var ( cases = []testCase{ { - service: path.ExchangeService, + srs: []path.ServiceResource{{ + Service: path.ExchangeService, + ProtectedResource: user, + }}, category: path.EmailCategory, expected: assert.Falsef, }, { - service: path.ExchangeService, + srs: []path.ServiceResource{{ + Service: path.ExchangeService, + ProtectedResource: user, + }}, category: path.ContactsCategory, expected: assert.Falsef, }, { - service: path.ExchangeService, + srs: []path.ServiceResource{{ + Service: path.ExchangeService, + ProtectedResource: user, + }}, category: path.EventsCategory, expected: assert.Falsef, }, { - service: path.OneDriveService, + srs: []path.ServiceResource{{ + Service: path.OneDriveService, + ProtectedResource: user, + }}, category: path.FilesCategory, expected: assert.Truef, }, { - service: path.SharePointService, + srs: []path.ServiceResource{{ + Service: path.SharePointService, + ProtectedResource: user, + }}, category: path.LibrariesCategory, expected: assert.Truef, }, { - service: path.SharePointService, + srs: []path.ServiceResource{{ + Service: path.SharePointService, + ProtectedResource: user, + }}, category: path.ListsCategory, expected: assert.Falsef, }, { - service: path.SharePointService, + srs: []path.ServiceResource{{ + Service: path.SharePointService, + ProtectedResource: user, + }}, category: path.PagesCategory, expected: assert.Falsef, }, + { + srs: []path.ServiceResource{ + { + Service: path.OneDriveService, + ProtectedResource: user, + }, + { + Service: path.ExchangeService, + ProtectedResource: user, + }, + }, + category: path.EventsCategory, + expected: assert.Falsef, + }, + { + srs: []path.ServiceResource{ + { + Service: path.ExchangeService, + ProtectedResource: user, + }, + { + Service: path.OneDriveService, + ProtectedResource: user, + }, + }, + category: path.FilesCategory, + expected: assert.Truef, + }, } ) @@ -87,21 +136,26 @@ func TestMetadataUnitSuite(t *testing.T) { func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() { for _, test := range cases { for _, ext := range metaSuffixes { - suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { + name := []string{} + + for _, sr := range test.srs { + name = append(name, sr.Service.String()) + } + + name = append(name, test.category.String(), ext) + + suite.Run(strings.Join(name, " "), func() { t := suite.T() p, err := path.Build( tenant, - []path.ServiceResource{{ - Service: test.service, - ProtectedResource: user, - }}, + test.srs, test.category, true, "file"+ext) require.NoError(t, err, clues.ToCore(err)) - test.expected(t, metadata.IsMetadataFile(p), "extension %s", ext) + test.expected(t, metadata.IsMetadataFilePath(p), "extension %s", ext) }) } } @@ -110,21 +164,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() { func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_NotMetaSuffixes() { for _, test := range cases { for _, ext := range notMetaSuffixes { - suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { + name := []string{} + + for _, sr := range test.srs { + name = append(name, sr.Service.String()) + } + + name = append(name, test.category.String(), ext) + + suite.Run(strings.Join(name, " "), func() { t := suite.T() p, err := path.Build( tenant, - []path.ServiceResource{{ - Service: test.service, - ProtectedResource: user, - }}, + test.srs, test.category, true, "file"+ext) require.NoError(t, err, clues.ToCore(err)) - assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext) + assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext) }) } } @@ -135,21 +194,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Directories() { for _, test := range cases { for _, ext := range suffixes { - suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { + name := []string{} + + for _, sr := range test.srs { + name = append(name, sr.Service.String()) + } + + name = append(name, test.category.String(), ext) + + suite.Run(strings.Join(name, " "), func() { t := suite.T() p, err := path.Build( tenant, - []path.ServiceResource{{ - Service: test.service, - ProtectedResource: user, - }}, + test.srs, test.category, false, "file"+ext) require.NoError(t, err, clues.ToCore(err)) - assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext) + assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext) }) } } diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index fee6cc1f5..39d2f6606 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -865,7 +865,6 @@ func compareItem( t *testing.T, colPath path.Path, expected map[string][]byte, - service path.ServiceType, category path.CategoryType, item data.Stream, mci m365Stub.ConfigInfo, @@ -875,7 +874,11 @@ func compareItem( assert.NotZero(t, mt.ModTime()) } - switch service { + // assume the last service in the path is the data owner + srs := colPath.ServiceResources() + lastService := srs[len(srs)-1].Service + + switch lastService { case path.ExchangeService: switch category { case path.EmailCategory: @@ -900,7 +903,7 @@ func compareItem( return compareDriveItem(t, expected, item, mci, rootDir) default: - assert.FailNowf(t, "unexpected service: %s", service.String()) + assert.FailNowf(t, "unexpected service: %s", lastService.String()) } return true @@ -929,9 +932,12 @@ func checkHasCollections( fp := g.FullPath() loc := g.(data.LocationPather).LocationPath() + // take the last service, since it should be the one owning data + srs := fp.ServiceResources() + service := srs[len(srs)-1].Service - if fp.Service() == path.OneDriveService || - (fp.Service() == path.SharePointService && fp.Category() == path.LibrariesCategory) { + if service == path.OneDriveService || + (service == path.SharePointService && fp.Category() == path.LibrariesCategory) { dp, err := path.ToDrivePath(fp) if !assert.NoError(t, err, clues.ToCore(err)) { continue @@ -971,11 +977,12 @@ func checkCollections( for _, returned := range got { var ( hasItems bool - service = returned.FullPath().Service() category = returned.FullPath().Category() expectedColData = expected[returned.FullPath().String()] folders = returned.FullPath().Elements() rootDir = folders[len(folders)-1] == mci.RestoreCfg.Location + srs = returned.FullPath().ServiceResources() + lastService = srs[len(srs)-1].Service ) // Need to iterate through all items even if we don't expect to find a match @@ -987,9 +994,9 @@ func checkCollections( // is for actual pull items. // TODO(ashmrtn): Should probably eventually check some data in metadata // collections. - if service == path.ExchangeMetadataService || - service == path.OneDriveMetadataService || - service == path.SharePointMetadataService { + if lastService == path.ExchangeMetadataService || + lastService == path.OneDriveMetadataService || + lastService == path.SharePointMetadataService { skipped++ continue } @@ -1005,7 +1012,6 @@ func checkCollections( t, returned.FullPath(), expectedColData, - service, category, item, mci, diff --git a/src/internal/m365/service/exchange/backup_test.go b/src/internal/m365/service/exchange/backup_test.go index acee5119e..e57cf2ce9 100644 --- a/src/internal/m365/service/exchange/backup_test.go +++ b/src/internal/m365/service/exchange/backup_test.go @@ -491,7 +491,9 @@ func (suite *BackupIntgSuite) TestMailFetch() { require.NoError(t, err, clues.ToCore(err)) for _, c := range collections { - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { continue } @@ -577,7 +579,9 @@ func (suite *BackupIntgSuite) TestDelta() { var metadata data.BackupCollection for _, coll := range collections { - if coll.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + coll.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadata = coll } } @@ -611,7 +615,9 @@ func (suite *BackupIntgSuite) TestDelta() { // Delta usage is commented out at the moment, anyway. So this is currently // a sanity check that the minimum behavior won't break. for _, coll := range collections { - if coll.FullPath().Service() != path.ExchangeMetadataService { + if !path.ServiceResourcesMatchServices( + coll.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { ec, ok := coll.(*Collection) require.True(t, ok, "collection is *Collection") assert.NotNil(t, ec) @@ -666,7 +672,9 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() { ctx, flush := tester.NewContext(t) defer flush() - isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService + isMetadata := path.ServiceResourcesMatchServices( + edc.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) streamChannel := edc.Items(ctx, fault.New(true)) // Verify that each message can be restored @@ -744,7 +752,9 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() { require.GreaterOrEqual(t, 2, len(edcs), "expected 1 <= num collections <= 2") for _, edc := range edcs { - isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService + isMetadata := path.ServiceResourcesMatchServices( + edc.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) count := 0 for stream := range edc.Items(ctx, fault.New(true)) { @@ -874,7 +884,10 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() { for _, edc := range collections { var isMetadata bool - if edc.FullPath().Service() != path.ExchangeMetadataService { + // FIXME: this doesn't seem right, it's saying "if not metadata, isMetadata = true" + if !path.ServiceResourcesMatchServices( + edc.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { isMetadata = true assert.Equal(t, test.expected, edc.FullPath().Folder(false)) } else { @@ -1140,7 +1153,9 @@ func (suite *CollectionPopulationSuite) TestPopulateCollections() { deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0 for _, c := range collections { - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ continue } @@ -1491,7 +1506,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D continue } - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c) continue @@ -1650,7 +1667,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_r deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0 for _, c := range collections { - if c.FullPath().Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ continue } @@ -2082,7 +2101,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i require.NotNil(t, p) - if p.Service() == path.ExchangeMetadataService { + if path.ServiceResourcesMatchServices( + c.FullPath().ServiceResources(), + []path.ServiceType{path.ExchangeMetadataService}) { metadatas++ continue } diff --git a/src/internal/m365/service/groups/restore.go b/src/internal/m365/service/groups/restore.go index e36b3d7df..5512df4c6 100644 --- a/src/internal/m365/service/groups/restore.go +++ b/src/internal/m365/service/groups/restore.go @@ -57,7 +57,9 @@ func ConsumeRestoreCollections( ictx = clues.Add(ctx, "category", category, "restore_location", clues.Hide(rcc.RestoreConfig.Location), - "protected_resource", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owners", clues.Hide( + path.ServiceResourcesToResources( + dc.FullPath().ServiceResources())), "full_path", dc.FullPath()) ) diff --git a/src/internal/m365/service/sharepoint/restore.go b/src/internal/m365/service/sharepoint/restore.go index 35e1c67cd..12aaad60e 100644 --- a/src/internal/m365/service/sharepoint/restore.go +++ b/src/internal/m365/service/sharepoint/restore.go @@ -61,7 +61,9 @@ func ConsumeRestoreCollections( ictx = clues.Add(ctx, "category", category, "restore_location", clues.Hide(rcc.RestoreConfig.Location), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owners", clues.Hide( + path.ServiceResourcesToResources( + dc.FullPath().ServiceResources())), "full_path", dc.FullPath()) ) diff --git a/src/internal/m365/stub/stub.go b/src/internal/m365/stub/stub.go index 133fc5ff1..c0d4459c1 100644 --- a/src/internal/m365/stub/stub.go +++ b/src/internal/m365/stub/stub.go @@ -7,7 +7,7 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + "github.com/alcionai/corso/src/internal/m365/graph/metadata" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" @@ -63,9 +63,11 @@ func GetCollectionsAndExpected( for _, owner := range config.ResourceOwners { numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo( - config.Service, config.Tenant, - owner, + []path.ServiceResource{{ + Service: config.Service, + ProtectedResource: owner, + }}, config.RestoreCfg, testCollections, backupVersion) @@ -128,9 +130,7 @@ func CollectionsForInfo( baseExpected[info.Items[i].LookupKey] = info.Items[i].Data // We do not count metadata files against item count - if backupVersion > 0 && - (service == path.OneDriveService || service == path.SharePointService) && - metadata.HasMetaSuffix(info.Items[i].Name) { + if backupVersion > 0 && metadata.IsMetadataFile(srs, info.Category, info.Items[i].Name) { continue } @@ -165,9 +165,13 @@ func backupOutputPathFromRestore( inputPath path.Path, ) (path.Path, error) { base := []string{restoreCfg.Location} + srs := inputPath.ServiceResources() + // only the last service is checked, because that should be the service + // whose data is stored.. + lastService := srs[len(srs)-1].Service // OneDrive has leading information like the drive ID. - if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService { + if lastService == path.OneDriveService || lastService == path.SharePointService { folders := inputPath.Folders() base = append(append([]string{}, folders[:3]...), restoreCfg.Location) @@ -176,14 +180,13 @@ func backupOutputPathFromRestore( } } - if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory { + if lastService == path.ExchangeService && inputPath.Category() == path.EmailCategory { base = append(base, inputPath.Folders()...) } return path.Build( inputPath.Tenant(), - inputPath.ResourceOwner(), - inputPath.Service(), + inputPath.ServiceResources(), inputPath.Category(), false, base...) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index dbe7d0d5c..2dead366f 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -566,7 +566,10 @@ func getNewPathRefs( // TODO(ashmrtn): In the future we can remove this first check as we'll be // able to assume we always have the location in the previous entry. We'll end // up doing some extra parsing, but it will simplify this code. - if repoRef.Service() == path.ExchangeService { + // + // Currently safe to check only the 0th SR, since exchange had no subservices + // at the time of the locations addition. + if repoRef.ServiceResources()[0].Service == path.ExchangeService { newPath, newLoc, err := dataFromBackup.GetNewPathRefs( repoRef.ToBuilder(), entry.Modified(), diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index d5e0ee3f3..5538c4ce5 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -266,6 +266,7 @@ func makePath(t *testing.T, elements []string, isItem bool) path.Path { return p } +// FIXME: out of date, does not contain sharepoint support func makeDetailsEntry( t *testing.T, p path.Path, @@ -290,7 +291,10 @@ func makeDetailsEntry( Updated: updated, } - switch p.Service() { + srs := p.ServiceResources() + lastService := srs[len(srs)-1].Service + + switch lastService { case path.ExchangeService: if p.Category() != path.EmailCategory { assert.FailNowf( @@ -321,7 +325,7 @@ func makeDetailsEntry( assert.FailNowf( t, "service %s not supported in helper function", - p.Service().String()) + lastService.String()) } return res @@ -529,6 +533,23 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections fault.New(true)) } +// makeElements allows creation of repoRefs that wouldn't +// pass paths package validators. +func makeElements( + tenant string, + srs []path.ServiceResource, + cat path.CategoryType, + suffix ...string, +) path.Elements { + elems := append( + path.Elements{tenant}, + path.ServiceResourcesToElements(srs)...) + elems = append(elems, cat.String()) + elems = append(elems, suffix...) + + return elems +} + func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() { var ( tenant = "a-tenant" @@ -709,17 +730,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems DetailsModel: details.DetailsModel{ Entries: []details.Entry{ { - RepoRef: stdpath.Join( - append( - []string{ - itemPath1.Tenant(), - itemPath1.Service().String(), - itemPath1.ResourceOwner(), - path.UnknownCategory.String(), - }, - itemPath1.Folders()..., - )..., - ), + RepoRef: stdpath.Join(makeElements( + itemPath1.Tenant(), + itemPath1.ServiceResources(), + path.UnknownCategory, + itemPath1.Folders()...)...), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -740,16 +755,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems res := newMockDetailsMergeInfoer() p := makePath( suite.T(), - []string{ + makeElements( itemPath1.Tenant(), - path.OneDriveService.String(), - itemPath1.ResourceOwner(), - path.FilesCategory.String(), + itemPath1.ServiceResources(), + path.FilesCategory, "personal", - "item1", - }, - true, - ) + "item1"), + true) res.add(itemPath1, p, nil) diff --git a/src/internal/streamstore/streamstore.go b/src/internal/streamstore/streamstore.go index 6f5918c81..49d9f16e9 100644 --- a/src/internal/streamstore/streamstore.go +++ b/src/internal/streamstore/streamstore.go @@ -198,8 +198,13 @@ func collect( service path.ServiceType, col Collectable, ) (data.BackupCollection, error) { + srs := []path.ServiceResource{{ + Service: service, + ProtectedResource: col.purpose, + }} + // construct the path of the container - p, err := path.Builder{}.ToStreamStorePath(tenantID, col.purpose, service, false) + p, err := path.Builder{}.ToStreamStorePath(tenantID, srs, false) if err != nil { return nil, clues.Stack(err).WithClues(ctx) } @@ -257,10 +262,15 @@ func read( rer inject.RestoreProducer, errs *fault.Bus, ) error { + srs := []path.ServiceResource{{ + Service: service, + ProtectedResource: col.purpose, + }} + // construct the path of the container p, err := path.Builder{}. Append(col.itemName). - ToStreamStorePath(tenantID, col.purpose, service, true) + ToStreamStorePath(tenantID, srs, true) if err != nil { return clues.Stack(err).WithClues(ctx) } diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index a8cb6222a..1bd380286 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -31,7 +31,7 @@ func newDataLayerResourcePath( cat CategoryType, isItem bool, ) dataLayerResourcePath { - pfx := append([]string{tenant}, serviceResourcesToElements(srs)...) + pfx := append([]string{tenant}, ServiceResourcesToElements(srs)...) pfx = append(pfx, cat.String()) return dataLayerResourcePath{ diff --git a/src/pkg/path/service_resource.go b/src/pkg/path/service_resource.go index 77317d029..037011a24 100644 --- a/src/pkg/path/service_resource.go +++ b/src/pkg/path/service_resource.go @@ -2,6 +2,7 @@ package path import ( "github.com/alcionai/clues" + "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/tform" @@ -84,11 +85,13 @@ func ServiceResourcesToResources(srs []ServiceResource) []string { return prs } -// --------------------------------------------------------------------------- -// Unexported Helpers -// --------------------------------------------------------------------------- +func ServiceResourcesMatchServices(srs []ServiceResource, sts []ServiceType) bool { + return slices.EqualFunc(srs, sts, func(sr ServiceResource, st ServiceType) bool { + return sr.Service == st + }) +} -func serviceResourcesToElements(srs []ServiceResource) Elements { +func ServiceResourcesToElements(srs []ServiceResource) Elements { es := make(Elements, 0, len(srs)*2) for _, tuple := range srs { @@ -99,6 +102,10 @@ func serviceResourcesToElements(srs []ServiceResource) Elements { return es } +// --------------------------------------------------------------------------- +// Unexported Helpers +// --------------------------------------------------------------------------- + // elementsToServiceResources turns as many pairs of elems as possible // into ServiceResource tuples. Elems must begin with a service, but // may contain more entries than there are service/resource pairs. diff --git a/src/pkg/path/service_resource_test.go b/src/pkg/path/service_resource_test.go index 97139c49a..6eab81350 100644 --- a/src/pkg/path/service_resource_test.go +++ b/src/pkg/path/service_resource_test.go @@ -157,7 +157,7 @@ func (suite *ServiceResourceUnitSuite) TestServiceResourceToElements() { suite.Run(test.name, func() { t := suite.T() - result := serviceResourcesToElements(test.srs) + result := ServiceResourcesToElements(test.srs) // not ElementsMatch, order matters assert.Equal(t, test.expect, result)