diff --git a/src/internal/kopia/backup_bases_test.go b/src/internal/kopia/backup_bases_test.go index faa402162..968228191 100644 --- a/src/internal/kopia/backup_bases_test.go +++ b/src/internal/kopia/backup_bases_test.go @@ -220,7 +220,10 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { reasons := make([]identity.Reasoner, 0, len(i.cat)) for _, c := range i.cat { - reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) + reasons = append(reasons, NewReason( + "", + []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, + c)) } m := makeManifest(baseID, "", "b"+baseID, reasons...) @@ -245,7 +248,10 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { reasons := make([]identity.Reasoner, 0, len(i.cat)) for _, c := range i.cat { - reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) + reasons = append(reasons, NewReason( + "", + []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, + c)) } m := makeManifest(baseID, "", "a"+baseID, reasons...) @@ -480,7 +486,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { ro := "resource_owner" makeMan := func(pct path.CategoryType, id, incmpl, bID string) ManifestEntry { - r := NewReason("", ro, path.ExchangeService, pct) + r := NewReason("", []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, pct) return makeManifest(id, incmpl, bID, r) } @@ -690,11 +696,17 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, - NewReason("", ro, path.ExchangeService, path.ContactsCategory)) + NewReason( + "", + []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, + path.ContactsCategory)) res.assistBases[0].Reasons = append( res.assistBases[0].Reasons, - NewReason("", ro, path.ExchangeService, path.ContactsCategory)) + NewReason( + "", + []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, + path.ContactsCategory)) return res }(), @@ -702,11 +714,17 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, - NewReason("", ro, path.ExchangeService, path.ContactsCategory)) + NewReason( + "", + []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, + path.ContactsCategory)) res.assistBases[0].Reasons = append( res.assistBases[0].Reasons, - NewReason("", ro, path.ExchangeService, path.ContactsCategory)) + NewReason( + "", + []path.ServiceResource{{ProtectedResource: ro, Service: path.ExchangeService}}, + path.ContactsCategory)) res.assistBases = append(res.mergeBases, res.assistBases...) diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index a53cb52cf..cb2bab5ea 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -31,15 +31,14 @@ const ( ) func NewReason( - tenant, resource string, - service path.ServiceType, + tenant string, + srs []path.ServiceResource, category path.CategoryType, ) identity.Reasoner { return reason{ - tenant: tenant, - resource: resource, - service: service, - category: category, + tenant: tenant, + serviceResources: srs, + category: category, } } @@ -47,22 +46,17 @@ type reason struct { // tenant appears here so that when this is moved to an inject package nothing // needs changed. However, kopia itself is blind to the fields in the reason // struct and relies on helper functions to get the information it needs. - tenant string - resource string - service path.ServiceType - category path.CategoryType + tenant string + serviceResources []path.ServiceResource + category path.CategoryType } func (r reason) Tenant() string { return r.tenant } -func (r reason) ProtectedResource() string { - return r.resource -} - -func (r reason) Service() path.ServiceType { - return r.service +func (r reason) ServiceResources() []path.ServiceResource { + return r.ServiceResources() } func (r reason) Category() path.CategoryType { @@ -72,8 +66,7 @@ func (r reason) Category() path.CategoryType { func (r reason) SubtreePath() (path.Path, error) { p, err := path.BuildPrefix( r.Tenant(), - r.ProtectedResource(), - r.Service(), + r.ServiceResources(), r.Category()) return p, clues.Wrap(err, "building path").OrNil() @@ -88,7 +81,13 @@ func tagKeys(r identity.Reasoner) []string { // reasonKey returns the concatenation of the ProtectedResource, Service, and Category. func reasonKey(r identity.Reasoner) string { - return r.ProtectedResource() + r.Service().String() + r.Category().String() + var rk string + + for _, sr := range r.ServiceResources() { + rk += sr.ProtectedResource + sr.Service.String() + } + + return rk + r.Category().String() } type BackupEntry struct { @@ -412,7 +411,7 @@ func (b *baseFinder) FindBases( for _, searchReason := range reasons { ictx := clues.Add( ctx, - "search_service", searchReason.Service().String(), + "search_service", path.ServiceResourcesToServices(searchReason.ServiceResources()), "search_category", searchReason.Category().String()) logger.Ctx(ictx).Info("searching for previous manifests") diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go index d1b0742fc..10d42a584 100644 --- a/src/internal/kopia/base_finder_test.go +++ b/src/internal/kopia/base_finder_test.go @@ -47,22 +47,22 @@ var ( testAllUsersAllCats = []identity.Reasoner{ // User1 email and events. - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), // User2 email and events. - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EventsCategory), // User3 email and events. - NewReason("", testUser3, path.ExchangeService, path.EmailCategory), - NewReason("", testUser3, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EventsCategory), } testAllUsersMail = []identity.Reasoner{ - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), - NewReason("", testUser3, path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EmailCategory), } testUser1Mail = []identity.Reasoner{ - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), } ) @@ -294,7 +294,7 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() { bg: mockEmptyModelGetter{}, } reasons := []identity.Reasoner{ - NewReason("", "a-user", path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: "a-user", Service: path.ExchangeService}}, path.EmailCategory), } bb := bf.FindBases(ctx, reasons, nil) @@ -313,7 +313,7 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { bg: mockEmptyModelGetter{}, } reasons := []identity.Reasoner{ - NewReason("", "a-user", path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: "a-user", Service: path.ExchangeService}}, path.EmailCategory), } bb := bf.FindBases(ctx, reasons, nil) @@ -602,26 +602,26 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { }, expectedBaseReasons: map[int][]identity.Reasoner{ 0: { - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), - NewReason("", testUser3, path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EmailCategory), }, 1: { - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), - NewReason("", testUser3, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EventsCategory), }, }, expectedAssistManifestReasons: map[int][]identity.Reasoner{ 0: { - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), - NewReason("", testUser3, path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EmailCategory), }, 1: { - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), - NewReason("", testUser3, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser3, Service: path.ExchangeService}}, path.EventsCategory), }, }, backupData: []backupInfo{ @@ -667,36 +667,36 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { }, expectedBaseReasons: map[int][]identity.Reasoner{ 2: { - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EventsCategory), }, }, expectedAssistManifestReasons: map[int][]identity.Reasoner{ 0: { - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EventsCategory), }, 1: { - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), }, 2: { - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EventsCategory), }, }, expectedAssistReasons: map[int][]identity.Reasoner{ 0: { - NewReason("", testUser1, path.ExchangeService, path.EventsCategory), - NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EventsCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EventsCategory), }, 1: { - NewReason("", testUser1, path.ExchangeService, path.EmailCategory), - NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser1, Service: path.ExchangeService}}, path.EmailCategory), + NewReason("", []path.ServiceResource{{ProtectedResource: testUser2, Service: path.ExchangeService}}, path.EmailCategory), }, }, backupData: []backupInfo{ diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 2d815594f..2dd1c8f39 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -196,14 +196,16 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) { return } + ctx := clues.Add(cp.ctx, + "services", path.ServiceResourcesToResources(d.repoPath.ServiceResources()), + "category", d.repoPath.Category().String()) + // These items were sourced from a base snapshot or were cached in kopia so we // never had to materialize their details in-memory. if d.info == nil || d.cached { if d.prevPath == nil { - cp.errs.AddRecoverable(cp.ctx, clues.New("item sourced from previous backup with no previous path"). - With( - "service", d.repoPath.Service().String(), - "category", d.repoPath.Category().String()). + cp.errs.AddRecoverable(ctx, clues.New("item sourced from previous backup with no previous path"). + WithClues(ctx). Label(fault.LabelForceNoBackupCreation)) return @@ -219,9 +221,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) { d.locationPath) if err != nil { cp.errs.AddRecoverable(cp.ctx, clues.Wrap(err, "adding item to merge list"). - With( - "service", d.repoPath.Service().String(), - "category", d.repoPath.Category().String()). + WithClues(ctx). Label(fault.LabelForceNoBackupCreation)) } @@ -235,9 +235,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) { *d.info) if err != nil { cp.errs.AddRecoverable(cp.ctx, clues.New("adding item to details"). - With( - "service", d.repoPath.Service().String(), - "category", d.repoPath.Category().String()). + WithClues(ctx). Label(fault.LabelForceNoBackupCreation)) return diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 582c3ff78..46e2af94c 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -724,8 +724,10 @@ func TestKopiaIntegrationSuite(t *testing.T) { func (suite *KopiaIntegrationSuite) SetupSuite() { tmp, err := path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, testInboxDir) @@ -736,8 +738,10 @@ func (suite *KopiaIntegrationSuite) SetupSuite() { tmp, err = path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, testArchiveDir) @@ -804,14 +808,12 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { reasons := []identity.Reasoner{ NewReason( testTenant, - suite.storePath1.ResourceOwner(), - suite.storePath1.Service(), + suite.storePath1.ServiceResources(), suite.storePath1.Category(), ), NewReason( testTenant, - suite.storePath2.ResourceOwner(), - suite.storePath2.Service(), + suite.storePath2.ServiceResources(), suite.storePath2.Category(), ), } @@ -1052,8 +1054,10 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { tmp, err := path.Build( testTenant, - testUser, - path.OneDriveService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.OneDriveService, + }}, path.FilesCategory, false, testInboxDir) @@ -1079,8 +1083,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { reasons := []identity.Reasoner{ NewReason( testTenant, - storePath.ResourceOwner(), - storePath.Service(), + storePath.ServiceResources(), storePath.Category()), } @@ -1258,7 +1261,13 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { w := &Wrapper{k} - r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) + r := NewReason( + testTenant, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, + path.EmailCategory) dc1 := exchMock.NewCollection(suite.storePath1, suite.locPath1, 1) dc2 := exchMock.NewCollection(suite.storePath2, suite.locPath2, 1) @@ -1347,7 +1356,13 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { loc1 := path.Builder{}.Append(suite.storePath1.Folders()...) loc2 := path.Builder{}.Append(suite.storePath2.Folders()...) - r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) + r := NewReason( + testTenant, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, + path.EmailCategory) collections := []data.BackupCollection{ &mockBackupCollection{ @@ -1507,8 +1522,10 @@ func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) { func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { tmp, err := path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, testInboxDir) @@ -1518,8 +1535,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { tmp, err = path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, testArchiveDir) @@ -1619,7 +1638,13 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { collections = append(collections, collection) } - r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) + r := NewReason( + testTenant, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, + path.EmailCategory) stats, deets, _, err := suite.w.ConsumeBackupCollections( suite.ctx, @@ -1657,7 +1682,13 @@ func (c *i64counter) Count(i int64) { } func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { - r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) + r := NewReason( + testTenant, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, + path.EmailCategory) man, err := suite.w.c.LoadSnapshot(suite.ctx, suite.snapshotID) require.NoError(suite.T(), err, "getting base snapshot: %v", clues.ToCore(err)) @@ -1800,8 +1831,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { doesntExist, err := path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, true, "subdir", "foo") @@ -1934,8 +1967,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_PathChanges() { rp1, err := path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, "corso_restore", "Inbox") @@ -1943,8 +1978,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Path rp2, err := path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, "corso_restore", "Archive") @@ -2057,8 +2094,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Fetc rp1, err := path.Build( testTenant, - testUser, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: testUser, + Service: path.ExchangeService, + }}, path.EmailCategory, false, "corso_restore", "Inbox") diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 5538c4ce5..f72a8e06f 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -460,13 +460,17 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections emailReason = kopia.NewReason( tenant, - resourceOwner, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: resourceOwner, + Service: path.ExchangeService, + }}, path.EmailCategory) contactsReason = kopia.NewReason( tenant, - resourceOwner, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: resourceOwner, + Service: path.ExchangeService, + }}, path.ContactsCategory) reasons = []identity.Reasoner{ @@ -617,13 +621,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems pathReason1 = kopia.NewReason( "", - itemPath1.ResourceOwner(), - itemPath1.Service(), + itemPath1.ServiceResources(), itemPath1.Category()) pathReason3 = kopia.NewReason( "", - itemPath3.ResourceOwner(), - itemPath3.Service(), + itemPath3.ServiceResources(), itemPath3.Category()) time1 = time.Now() @@ -1240,8 +1242,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde pathReason1 = kopia.NewReason( "", - itemPath1.ResourceOwner(), - itemPath1.Service(), + itemPath1.ServiceResources(), itemPath1.Category()) backup1 = kopia.BackupEntry{ diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index 95b313adc..073f205b9 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -145,8 +145,7 @@ func collectMetadata( Append(fn). ToServiceCategoryMetadataPath( tenantID, - reason.ProtectedResource(), - reason.Service(), + reason.ServiceResources(), reason.Category(), true) if err != nil { diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index 6d91b1e31..589d86f88 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -112,7 +112,13 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { name: "single reason, single file", manID: "single single", reasons: []identity.Reasoner{ - kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + tid, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -131,7 +137,13 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { name: "single reason, multiple files", manID: "single multi", reasons: []identity.Reasoner{ - kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + tid, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -150,8 +162,20 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { name: "multiple reasons, single file", manID: "multi single", reasons: []identity.Reasoner{ - kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), - kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + tid, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), + kopia.NewReason( + tid, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -173,8 +197,20 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { name: "multiple reasons, multiple file", manID: "multi multi", reasons: []identity.Reasoner{ - kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), - kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + tid, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), + kopia.NewReason( + tid, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -226,7 +262,13 @@ func buildReasons( for _, cat := range cats { reasons = append( reasons, - kopia.NewReason("", ro, service, cat)) + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: service, + }}, + cat)) } return reasons @@ -283,7 +325,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, rp: mockRestoreProducer{}, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, getMeta: false, assertErr: assert.NoError, @@ -304,7 +352,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, rp: mockRestoreProducer{}, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -332,8 +386,20 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), - kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -379,7 +445,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -409,7 +481,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, getMeta: true, dropAssist: true, @@ -438,7 +516,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -460,7 +544,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, rp: mockRestoreProducer{err: assert.AnError}, reasons: []identity.Reasoner{ - kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.EmailCategory), }, getMeta: true, assertErr: assert.Error, @@ -566,14 +656,18 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb emailReason := kopia.NewReason( "", - ro, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, path.EmailCategory) fbEmailReason := kopia.NewReason( "", - fbro, - path.ExchangeService, + []path.ServiceResource{{ + ProtectedResource: fbro, + Service: path.ExchangeService, + }}, path.EmailCategory) table := []struct { @@ -880,11 +974,23 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb }, reasons: []identity.Reasoner{ emailReason, - kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, fallbackReasons: []identity.Reasoner{ fbEmailReason, - kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: fbro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -916,7 +1022,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb }, reasons: []identity.Reasoner{emailReason}, fallbackReasons: []identity.Reasoner{ - kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: fbro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -951,11 +1063,23 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb }, reasons: []identity.Reasoner{ emailReason, - kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: ro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, fallbackReasons: []identity.Reasoner{ fbEmailReason, - kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), + kopia.NewReason( + "", + []path.ServiceResource{{ + ProtectedResource: fbro, + Service: path.ExchangeService, + }}, + path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, diff --git a/src/pkg/backup/identity/identity.go b/src/pkg/backup/identity/identity.go index 0f0d77416..97aa5f03c 100644 --- a/src/pkg/backup/identity/identity.go +++ b/src/pkg/backup/identity/identity.go @@ -7,8 +7,7 @@ import "github.com/alcionai/corso/src/pkg/path" // categories which are held within the backup. type Reasoner interface { Tenant() string - ProtectedResource() string - Service() path.ServiceType + ServiceResources() []path.ServiceResource Category() path.CategoryType // SubtreePath returns the path prefix for data in existing backups that have // parameters (tenant, protected resourced, etc) that match this Reasoner. diff --git a/src/pkg/path/service_resource.go b/src/pkg/path/service_resource.go index 037011a24..35c4ea38b 100644 --- a/src/pkg/path/service_resource.go +++ b/src/pkg/path/service_resource.go @@ -85,6 +85,16 @@ func ServiceResourcesToResources(srs []ServiceResource) []string { return prs } +func ServiceResourcesToServices(srs []ServiceResource) []ServiceType { + sts := make([]ServiceType, len(srs)) + + for i := range srs { + sts[i] = srs[i].Service + } + + return sts +} + func ServiceResourcesMatchServices(srs []ServiceResource, sts []ServiceType) bool { return slices.EqualFunc(srs, sts, func(sr ServiceResource, st ServiceType) bool { return sr.Service == st diff --git a/src/pkg/selectors/reasons.go b/src/pkg/selectors/reasons.go index a27d87f52..637c8c6af 100644 --- a/src/pkg/selectors/reasons.go +++ b/src/pkg/selectors/reasons.go @@ -15,22 +15,17 @@ import ( var _ identity.Reasoner = &backupReason{} type backupReason struct { - category path.CategoryType - resource string - service path.ServiceType - tenant string + category path.CategoryType + serviceResources []path.ServiceResource + tenant string } func (br backupReason) Tenant() string { return br.tenant } -func (br backupReason) ProtectedResource() string { - return br.resource -} - -func (br backupReason) Service() path.ServiceType { - return br.service +func (br backupReason) ServiceResources() []path.ServiceResource { + return br.serviceResources } func (br backupReason) Category() path.CategoryType { @@ -40,13 +35,18 @@ func (br backupReason) Category() path.CategoryType { func (br backupReason) SubtreePath() (path.Path, error) { return path.BuildPrefix( br.tenant, - br.resource, - br.service, + br.serviceResources, br.category) } func (br backupReason) key() string { - return br.category.String() + br.resource + br.service.String() + br.tenant + var k string + + for _, sr := range br.serviceResources { + k += sr.ProtectedResource + sr.Service.String() + } + + return br.category.String() + k + br.tenant } // --------------------------------------------------------------------------- diff --git a/src/pkg/selectors/reasons_test.go b/src/pkg/selectors/reasons_test.go index 92ff3b807..ec5fd459a 100644 --- a/src/pkg/selectors/reasons_test.go +++ b/src/pkg/selectors/reasons_test.go @@ -22,17 +22,17 @@ func TestReasonsUnitSuite(t *testing.T) { func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { var ( - tenantID = "tid" - exchange = path.ExchangeService.String() - email = path.EmailCategory.String() - contacts = path.ContactsCategory.String() + tenantID = "tid" + pstExchange = path.ExchangeService + exchange = pstExchange.String() + email = path.EmailCategory.String() + contacts = path.ContactsCategory.String() ) type expect struct { tenant string - resource string + serviceResources []path.ServiceResource category string - service string subtreePath string subtreePathHadErr bool } @@ -69,10 +69,12 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { useName: true, expect: []expect{ { - tenant: tenantID, - resource: "timbubba", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "timbubba", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("timbubba", email, exchange), }, }, @@ -86,10 +88,12 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { }, expect: []expect{ { - tenant: tenantID, - resource: "bubba", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "bubba", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("bubba", email, exchange), }, }, @@ -103,10 +107,12 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { }, expect: []expect{ { - tenant: tenantID, - resource: "tachoma dhaume", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "tachoma dhaume", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("tachoma dhaume", email, exchange), }, }, @@ -122,10 +128,12 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { }, expect: []expect{ { - tenant: tenantID, - resource: "vyng vang zoombah", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "vyng vang zoombah", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("vyng vang zoombah", email, exchange), }, }, @@ -140,10 +148,12 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { }, expect: []expect{ { - tenant: tenantID, - resource: "fat billie", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "fat billie", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("fat billie", email, exchange), }, }, @@ -158,10 +168,12 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { }, expect: []expect{ { - tenant: tenantID, - resource: "seathane", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "seathane", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("seathane", email, exchange), }, }, @@ -176,17 +188,21 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { }, expect: []expect{ { - tenant: tenantID, - resource: "perell", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "perell", + Service: pstExchange, + }}, category: email, - service: exchange, subtreePath: stpFor("perell", email, exchange), }, { - tenant: tenantID, - resource: "perell", + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "perell", + Service: pstExchange, + }}, category: contacts, - service: exchange, subtreePath: stpFor("perell", contacts, exchange), }, }, @@ -211,8 +227,7 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { results = append(results, expect{ tenant: r.Tenant(), - resource: r.ProtectedResource(), - service: r.Service().String(), + serviceResources: r.ServiceResources(), category: r.Category().String(), subtreePath: stpStr, subtreePathHadErr: err != nil, @@ -225,150 +240,103 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { } func (suite *ReasonsUnitSuite) TestReasonsFor_serviceChecks() { - var ( - tenantID = "tid" - exchange = path.ExchangeService.String() - email = path.EmailCategory.String() - contacts = path.ContactsCategory.String() - ) + var tenantID = "tid" type expect struct { tenant string - resource string + serviceResources []path.ServiceResource category string - service string subtreePath string subtreePathHadErr bool } - stpFor := func(resource, category, service string) string { - return path.Builder{}.Append(tenantID, service, resource, category).String() + stpFor := func( + resource string, + service path.ServiceType, + category path.CategoryType, + ) string { + return path.Builder{}.Append(tenantID, service.String(), resource, category.String()).String() } table := []struct { name string - sel func() ExchangeRestore + sel func() servicerCategorizerProvider useName bool expect []expect }{ { - name: "no scopes", - sel: func() ExchangeRestore { - return *NewExchangeRestore([]string{"timbo"}) - }, - expect: []expect{}, - }, - { - name: "only includes", - sel: func() ExchangeRestore { - sel := *NewExchangeRestore([]string{"bubba"}) + name: "exchange", + sel: func() servicerCategorizerProvider { + sel := *NewExchangeRestore([]string{"hadrian"}) sel.Include(sel.MailFolders(Any())) return sel }, expect: []expect{ { - tenant: tenantID, - resource: "bubba", - category: email, - service: exchange, - subtreePath: stpFor("bubba", email, exchange), + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "hadrian", + Service: path.ExchangeService, + }}, + category: path.EmailCategory.String(), + subtreePath: stpFor("hadrian", path.ExchangeService, path.EmailCategory), }, }, }, { - name: "only filters", - sel: func() ExchangeRestore { - sel := *NewExchangeRestore([]string{"tachoma dhaume"}) - sel.Filter(sel.MailFolders(Any())) + name: "onedrive", + sel: func() servicerCategorizerProvider { + sel := *NewOneDriveRestore([]string{"hella"}) + sel.Filter(sel.Folders(Any())) return sel }, expect: []expect{ { - tenant: tenantID, - resource: "tachoma dhaume", - category: email, - service: exchange, - subtreePath: stpFor("tachoma dhaume", email, exchange), + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "hella", + Service: path.OneDriveService, + }}, + category: path.FilesCategory.String(), + subtreePath: stpFor("hella", path.OneDriveService, path.FilesCategory), }, }, }, { - name: "duplicate includes and filters", - sel: func() ExchangeRestore { - sel := *NewExchangeRestore([]string{"vyng vang zoombah"}) - sel.Include(sel.MailFolders(Any())) - sel.Filter(sel.MailFolders(Any())) - + name: "sharepoint", + sel: func() servicerCategorizerProvider { + sel := *NewSharePointRestore([]string{"lem king"}) + sel.Include(sel.LibraryFolders(Any())) return sel }, expect: []expect{ { - tenant: tenantID, - resource: "vyng vang zoombah", - category: email, - service: exchange, - subtreePath: stpFor("vyng vang zoombah", email, exchange), + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "lem king", + Service: path.SharePointService, + }}, + category: path.EmailCategory.String(), + subtreePath: stpFor("lem king", path.SharePointService, path.LibrariesCategory), }, }, }, { - name: "duplicate includes", - sel: func() ExchangeRestore { - sel := *NewExchangeRestore([]string{"fat billie"}) - sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any())) - + name: "groups", + sel: func() servicerCategorizerProvider { + sel := *NewGroupsRestore([]string{"fero feritas"}) + sel.Include(sel.TODO(Any())) return sel }, expect: []expect{ { - tenant: tenantID, - resource: "fat billie", - category: email, - service: exchange, - subtreePath: stpFor("fat billie", email, exchange), - }, - }, - }, - { - name: "duplicate filters", - sel: func() ExchangeRestore { - sel := *NewExchangeRestore([]string{"seathane"}) - sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any())) - - return sel - }, - expect: []expect{ - { - tenant: tenantID, - resource: "seathane", - category: email, - service: exchange, - subtreePath: stpFor("seathane", email, exchange), - }, - }, - }, - { - name: "no duplicates", - sel: func() ExchangeRestore { - sel := *NewExchangeRestore([]string{"perell"}) - sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any())) - - return sel - }, - expect: []expect{ - { - tenant: tenantID, - resource: "perell", - category: email, - service: exchange, - subtreePath: stpFor("perell", email, exchange), - }, - { - tenant: tenantID, - resource: "perell", - category: contacts, - service: exchange, - subtreePath: stpFor("perell", contacts, exchange), + tenant: tenantID, + serviceResources: []path.ServiceResource{{ + ProtectedResource: "fero feritas", + Service: path.GroupsService, + }}, + category: path.EmailCategory.String(), + subtreePath: stpFor("fero feritas", path.GroupsService, path.LibrariesCategory), }, }, }, @@ -392,8 +360,7 @@ func (suite *ReasonsUnitSuite) TestReasonsFor_serviceChecks() { results = append(results, expect{ tenant: r.Tenant(), - resource: r.ProtectedResource(), - service: r.Service().String(), + serviceResources: r.ServiceResources(), category: r.Category().String(), subtreePath: stpStr, subtreePathHadErr: err != nil,