diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index d788a9a34..b384a23b4 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -436,12 +436,18 @@ func (c *Collections) Get( } // add metadata collections - service, category := c.handler.ServiceCat() + pathPrefix, err := c.handler.MetadataPathPrefix(c.tenantID) + if err != nil { + // It's safe to return here because the logic for starting an + // incremental backup should eventually find that the metadata files are + // empty/missing and default to a full backup. + logger.CtxErr(ctx, err).Info("making metadata collection path prefixes") + + return collections, canUsePreviousBackup, nil + } + md, err := graph.MakeMetadataCollection( - c.tenantID, - c.resourceOwner, - service, - category, + pathPrefix, []graph.MetadataCollectionEntry{ graph.NewMetadataEntry(graph.PreviousPathFileName, folderPaths), graph.NewMetadataEntry(graph.DeltaURLsFileName, deltaURLs), diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index 5604c4f42..085bbc0b7 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -1129,11 +1129,16 @@ func (suite *OneDriveCollectionsUnitSuite) TestDeserializeMetadata() { cols := []data.RestoreCollection{} for _, c := range test.cols { - mc, err := graph.MakeMetadataCollection( + pathPrefix, err := path.Builder{}.ToServiceCategoryMetadataPath( tenant, user, path.OneDriveService, path.FilesCategory, + false) + require.NoError(t, err, clues.ToCore(err)) + + mc, err := graph.MakeMetadataCollection( + pathPrefix, c(), func(*support.ControllerOperationStatus) {}) require.NoError(t, err, clues.ToCore(err)) @@ -2291,11 +2296,12 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { control.Options{ToggleFeatures: control.Toggles{}}) prevDelta := "prev-delta" + + pathPrefix, err := mbh.MetadataPathPrefix(tenant) + require.NoError(t, err, clues.ToCore(err)) + mc, err := graph.MakeMetadataCollection( - tenant, - user, - path.OneDriveService, - path.FilesCategory, + pathPrefix, []graph.MetadataCollectionEntry{ graph.NewMetadataEntry( graph.DeltaURLsFileName, diff --git a/src/internal/m365/collection/drive/group_handler.go b/src/internal/m365/collection/drive/group_handler.go index 585bf738d..386b3c696 100644 --- a/src/internal/m365/collection/drive/group_handler.go +++ b/src/internal/m365/collection/drive/group_handler.go @@ -1,6 +1,8 @@ package drive import ( + "github.com/alcionai/clues" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -50,6 +52,20 @@ func (h groupBackupHandler) PathPrefix( odConsts.RootPathDir) } +func (h groupBackupHandler) MetadataPathPrefix(tenantID string) (path.Path, error) { + p, err := path.Builder{}.ToServiceCategoryMetadataPath( + tenantID, + h.groupID, + h.service, + path.LibrariesCategory, + false) + if err != nil { + return nil, clues.Wrap(err, "making metadata path") + } + + return p, nil +} + func (h groupBackupHandler) CanonicalPath( folders *path.Builder, tenantID string, diff --git a/src/internal/m365/collection/drive/handlers.go b/src/internal/m365/collection/drive/handlers.go index 947f949ca..7b0064546 100644 --- a/src/internal/m365/collection/drive/handlers.go +++ b/src/internal/m365/collection/drive/handlers.go @@ -41,6 +41,9 @@ type BackupHandler interface { // the given values. PathPrefix(tenantID, driveID string) (path.Path, error) + // MetadataPathPrefix returns the prefix path for metadata + MetadataPathPrefix(tenantID string) (path.Path, error) + // CanonicalPath constructs the service and category specific path for // the given values. CanonicalPath(folders *path.Builder, tenantID string) (path.Path, error) diff --git a/src/internal/m365/collection/drive/item_handler.go b/src/internal/m365/collection/drive/item_handler.go index 16ae4dc3a..fa5a21252 100644 --- a/src/internal/m365/collection/drive/item_handler.go +++ b/src/internal/m365/collection/drive/item_handler.go @@ -54,6 +54,22 @@ func (h itemBackupHandler) PathPrefix( odConsts.RootPathDir) } +func (h itemBackupHandler) MetadataPathPrefix( + tenantID string, +) (path.Path, error) { + p, err := path.Builder{}.ToServiceCategoryMetadataPath( + tenantID, + h.userID, + path.OneDriveService, + path.FilesCategory, + false) + if err != nil { + return nil, clues.Wrap(err, "making metadata path") + } + + return p, nil +} + func (h itemBackupHandler) CanonicalPath( folders *path.Builder, tenantID string, diff --git a/src/internal/m365/collection/drive/library_handler.go b/src/internal/m365/collection/drive/library_handler.go index f098be8ea..02c3c6b68 100644 --- a/src/internal/m365/collection/drive/library_handler.go +++ b/src/internal/m365/collection/drive/library_handler.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -57,6 +58,22 @@ func (h libraryBackupHandler) PathPrefix( odConsts.RootPathDir) } +func (h libraryBackupHandler) MetadataPathPrefix( + tenantID string, +) (path.Path, error) { + p, err := path.Builder{}.ToServiceCategoryMetadataPath( + tenantID, + h.siteID, + h.service, + path.LibrariesCategory, + false) + if err != nil { + return nil, clues.Wrap(err, "making metadata path") + } + + return p, nil +} + func (h libraryBackupHandler) CanonicalPath( folders *path.Builder, tenantID string, diff --git a/src/internal/m365/collection/exchange/backup.go b/src/internal/m365/collection/exchange/backup.go index f5ebd1783..e97354ccd 100644 --- a/src/internal/m365/collection/exchange/backup.go +++ b/src/internal/m365/collection/exchange/backup.go @@ -266,11 +266,18 @@ func populateCollections( "num_paths_entries", len(currPaths), "num_deltas_entries", len(deltaURLs)) - col, err := graph.MakeMetadataCollection( + pathPrefix, err := path.Builder{}.ToServiceCategoryMetadataPath( qp.TenantID, qp.ProtectedResource.ID(), path.ExchangeService, qp.Category, + false) + if err != nil { + return nil, clues.Wrap(err, "making metadata path") + } + + col, err := graph.MakeMetadataCollection( + pathPrefix, []graph.MetadataCollectionEntry{ graph.NewMetadataEntry(graph.PreviousPathFileName, currPaths), graph.NewMetadataEntry(graph.DeltaURLsFileName, deltaURLs), diff --git a/src/internal/m365/collection/exchange/backup_test.go b/src/internal/m365/collection/exchange/backup_test.go index 6e4e20ba1..75b3f2bfe 100644 --- a/src/internal/m365/collection/exchange/backup_test.go +++ b/src/internal/m365/collection/exchange/backup_test.go @@ -299,10 +299,15 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() { graph.NewMetadataEntry(d.fileName, map[string]string{"key": d.value})) } - coll, err := graph.MakeMetadataCollection( + pathPrefix, err := path.Builder{}.ToServiceCategoryMetadataPath( "t", "u", path.ExchangeService, path.EmailCategory, + false) + require.NoError(t, err, "path prefix") + + coll, err := graph.MakeMetadataCollection( + pathPrefix, entries, func(cos *support.ControllerOperationStatus) {}, ) diff --git a/src/internal/m365/collection/site/backup.go b/src/internal/m365/collection/site/backup.go index f574ee4b5..0ce62c14e 100644 --- a/src/internal/m365/collection/site/backup.go +++ b/src/internal/m365/collection/site/backup.go @@ -43,9 +43,6 @@ func CollectLibraries( bpc.Options) ) - // TODO(meain): backup resource owner should be group id in case - // of group sharepoint site backup. As of now, we always use - // sharepoint site ids. odcs, canUsePreviousBackup, err := colls.Get(ctx, bpc.MetadataCollections, ssmb, errs) if err != nil { return nil, false, graph.Wrap(ctx, err, "getting library") diff --git a/src/internal/m365/graph/metadata_collection.go b/src/internal/m365/graph/metadata_collection.go index 12d668103..dabb1b7bb 100644 --- a/src/internal/m365/graph/metadata_collection.go +++ b/src/internal/m365/graph/metadata_collection.go @@ -63,9 +63,7 @@ func (mce MetadataCollectionEntry) toMetadataItem() (MetadataItem, error) { // containing all the provided metadata as a single json object. Returns // nil if the map does not have any entries. func MakeMetadataCollection( - tenant, resourceOwner string, - service path.ServiceType, - cat path.CategoryType, + pathPrefix path.Path, metadata []MetadataCollectionEntry, statusUpdater support.StatusUpdater, ) (data.BackupCollection, error) { @@ -73,16 +71,6 @@ func MakeMetadataCollection( return nil, nil } - p, err := path.Builder{}.ToServiceCategoryMetadataPath( - tenant, - resourceOwner, - service, - cat, - false) - if err != nil { - return nil, clues.Wrap(err, "making metadata path") - } - items := make([]MetadataItem, 0, len(metadata)) for _, md := range metadata { @@ -94,7 +82,7 @@ func MakeMetadataCollection( items = append(items, item) } - coll := NewMetadataCollection(p, items, statusUpdater) + coll := NewMetadataCollection(pathPrefix, items, statusUpdater) return coll, nil } diff --git a/src/internal/m365/graph/metadata_collection_test.go b/src/internal/m365/graph/metadata_collection_test.go index 64f544c19..d280c13ff 100644 --- a/src/internal/m365/graph/metadata_collection_test.go +++ b/src/internal/m365/graph/metadata_collection_test.go @@ -116,6 +116,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() { cat path.CategoryType metadata MetadataCollectionEntry collectionCheck assert.ValueAssertionFunc + pathPrefixCheck assert.ErrorAssertionFunc errCheck assert.ErrorAssertionFunc }{ { @@ -124,6 +125,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() { cat: path.EmailCategory, metadata: NewMetadataEntry("", nil), collectionCheck: assert.Nil, + pathPrefixCheck: assert.NoError, errCheck: assert.Error, }, { @@ -137,6 +139,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() { "hola": "mundo", }), collectionCheck: assert.NotNil, + pathPrefixCheck: assert.NoError, errCheck: assert.NoError, }, { @@ -150,7 +153,8 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() { "hola": "mundo", }), collectionCheck: assert.Nil, - errCheck: assert.Error, + pathPrefixCheck: assert.Error, + errCheck: assert.NoError, }, } @@ -161,11 +165,19 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() { ctx, flush := tester.NewContext(t) defer flush() - col, err := MakeMetadataCollection( + pathPrefix, err := path.Builder{}.ToServiceCategoryMetadataPath( tenant, user, test.service, test.cat, + false) + test.pathPrefixCheck(t, err, "path prefix") + if err != nil { + return + } + + col, err := MakeMetadataCollection( + pathPrefix, []MetadataCollectionEntry{test.metadata}, func(*support.ControllerOperationStatus) {}) diff --git a/src/internal/m365/service/onedrive/mock/handlers.go b/src/internal/m365/service/onedrive/mock/handlers.go index 248034f29..88f70f5fe 100644 --- a/src/internal/m365/service/onedrive/mock/handlers.go +++ b/src/internal/m365/service/onedrive/mock/handlers.go @@ -28,6 +28,9 @@ type BackupHandler struct { PathPrefixFn pathPrefixer PathPrefixErr error + MetadataPathPrefixFn metadataPathPrefixer + MetadataPathPrefixErr error + CanonPathFn canonPather CanonPathErr error @@ -52,16 +55,17 @@ func DefaultOneDriveBH(resourceOwner string) *BackupHandler { OneDrive: &details.OneDriveInfo{}, Extension: &details.ExtensionData{}, }, - GI: GetsItem{Err: clues.New("not defined")}, - GIP: GetsItemPermission{Err: clues.New("not defined")}, - PathPrefixFn: defaultOneDrivePathPrefixer, - CanonPathFn: defaultOneDriveCanonPather, - ResourceOwner: resourceOwner, - Service: path.OneDriveService, - Category: path.FilesCategory, - LocationIDFn: defaultOneDriveLocationIDer, - GetResps: []*http.Response{nil}, - GetErrs: []error{clues.New("not defined")}, + GI: GetsItem{Err: clues.New("not defined")}, + GIP: GetsItemPermission{Err: clues.New("not defined")}, + PathPrefixFn: defaultOneDrivePathPrefixer, + MetadataPathPrefixFn: defaultOneDriveMetadataPathPrefixer, + CanonPathFn: defaultOneDriveCanonPather, + ResourceOwner: resourceOwner, + Service: path.OneDriveService, + Category: path.FilesCategory, + LocationIDFn: defaultOneDriveLocationIDer, + GetResps: []*http.Response{nil}, + GetErrs: []error{clues.New("not defined")}, } } @@ -71,16 +75,17 @@ func DefaultSharePointBH(resourceOwner string) *BackupHandler { SharePoint: &details.SharePointInfo{}, Extension: &details.ExtensionData{}, }, - GI: GetsItem{Err: clues.New("not defined")}, - GIP: GetsItemPermission{Err: clues.New("not defined")}, - PathPrefixFn: defaultSharePointPathPrefixer, - CanonPathFn: defaultSharePointCanonPather, - ResourceOwner: resourceOwner, - Service: path.SharePointService, - Category: path.LibrariesCategory, - LocationIDFn: defaultSharePointLocationIDer, - GetResps: []*http.Response{nil}, - GetErrs: []error{clues.New("not defined")}, + GI: GetsItem{Err: clues.New("not defined")}, + GIP: GetsItemPermission{Err: clues.New("not defined")}, + PathPrefixFn: defaultSharePointPathPrefixer, + MetadataPathPrefixFn: defaultSharePointMetadataPathPrefixer, + CanonPathFn: defaultSharePointCanonPather, + ResourceOwner: resourceOwner, + Service: path.SharePointService, + Category: path.LibrariesCategory, + LocationIDFn: defaultSharePointLocationIDer, + GetResps: []*http.Response{nil}, + GetErrs: []error{clues.New("not defined")}, } } @@ -93,6 +98,15 @@ func (h BackupHandler) PathPrefix(tID, driveID string) (path.Path, error) { return pp, h.PathPrefixErr } +func (h BackupHandler) MetadataPathPrefix(tID string) (path.Path, error) { + pp, err := h.MetadataPathPrefixFn(tID, h.ResourceOwner) + if err != nil { + return nil, err + } + + return pp, h.PathPrefixErr +} + func (h BackupHandler) CanonicalPath(pb *path.Builder, tID string) (path.Path, error) { cp, err := h.CanonPathFn(pb, tID, h.ResourceOwner) if err != nil { @@ -159,7 +173,10 @@ var defaultSharePointCanonPather = func(pb *path.Builder, tID, ro string) (path. return pb.ToDataLayerSharePointPath(tID, ro, path.LibrariesCategory, false) } -type pathPrefixer func(tID, ro, driveID string) (path.Path, error) +type ( + pathPrefixer func(tID, ro, driveID string) (path.Path, error) + metadataPathPrefixer func(tID, ro string) (path.Path, error) +) var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, error) { return path.Build( @@ -173,6 +190,15 @@ var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, erro odConsts.RootPathDir) } +var defaultOneDriveMetadataPathPrefixer = func(tID, ro string) (path.Path, error) { + return path.Builder{}.ToServiceCategoryMetadataPath( + tID, + ro, + path.OneDriveService, + path.FilesCategory, + false) +} + var defaultSharePointPathPrefixer = func(tID, ro, driveID string) (path.Path, error) { return path.Build( tID, @@ -185,6 +211,15 @@ var defaultSharePointPathPrefixer = func(tID, ro, driveID string) (path.Path, er odConsts.RootPathDir) } +var defaultSharePointMetadataPathPrefixer = func(tID, ro string) (path.Path, error) { + return path.Builder{}.ToServiceCategoryMetadataPath( + tID, + ro, + path.SharePointService, + path.LibrariesCategory, + false) +} + type locationIDer func(string, ...string) details.LocationIDer var defaultOneDriveLocationIDer = func(driveID string, elems ...string) details.LocationIDer { diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index e8970ea1c..0247ef15d 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -1871,11 +1871,16 @@ func (suite *AssistBackupIntegrationSuite) TestBackupTypesForFailureModes() { cs := test.collFunc() - mc, err := graph.MakeMetadataCollection( + pathPrefix, err := path.Builder{}.ToServiceCategoryMetadataPath( tenantID, userID, path.OneDriveService, path.FilesCategory, + false) + require.NoError(t, err, clues.ToCore(err)) + + mc, err := graph.MakeMetadataCollection( + pathPrefix, makeMetadataCollectionEntries("url/1", driveID, folderID, tmp), func(*support.ControllerOperationStatus) {}) require.NoError(t, err, clues.ToCore(err)) @@ -2184,11 +2189,16 @@ func (suite *AssistBackupIntegrationSuite) TestExtensionsIncrementals() { cs := test.collFunc() - mc, err := graph.MakeMetadataCollection( + pathPrefix, err := path.Builder{}.ToServiceCategoryMetadataPath( tenantID, userID, path.OneDriveService, path.FilesCategory, + false) + require.NoError(t, err, clues.ToCore(err)) + + mc, err := graph.MakeMetadataCollection( + pathPrefix, makeMetadataCollectionEntries("url/1", driveID, folderID, tmp), func(*support.ControllerOperationStatus) {}) require.NoError(t, err, clues.ToCore(err))