From 9664afaa123bb6fe3d4fdb17b41289c0cb2b4ba0 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 19 Apr 2023 08:30:03 -0700 Subject: [PATCH] Restore with OneDrive/SharePoint ID (#3151) Allow restoring OneDrive and SharePoint items by M365 ID as well as file name and ShortRef Does create an edgecase where attempting to restore an item named the same as another item's M365 ID causes both items to be restored Moves the consts for OneDrive file suffixes to a different package so that the selectors package can also import them without a cycle --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * closes #3142 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 3 ++ src/cli/utils/testdata/opts.go | 18 +++++--- .../connector/graph/metadata/metadata.go | 4 +- .../connector/graph/metadata/metadata_test.go | 8 ++-- .../connector/graph_connector_helper_test.go | 13 +++--- .../graph_connector_onedrive_test.go | 43 ++++++++++--------- src/internal/connector/onedrive/collection.go | 16 ++----- .../connector/onedrive/collection_test.go | 7 +-- .../connector/onedrive/collections.go | 13 +++--- .../connector/onedrive/collections_test.go | 5 ++- .../connector/onedrive/metadata/consts.go | 13 ++++++ src/internal/connector/onedrive/permission.go | 5 ++- src/internal/connector/onedrive/restore.go | 15 ++++--- src/internal/kopia/wrapper_test.go | 8 ++-- src/pkg/repository/repository.go | 4 +- src/pkg/selectors/onedrive.go | 6 ++- src/pkg/selectors/onedrive_test.go | 2 +- src/pkg/selectors/sharepoint.go | 8 ++++ src/pkg/selectors/sharepoint_test.go | 2 +- 19 files changed, 112 insertions(+), 81 deletions(-) create mode 100644 src/internal/connector/onedrive/metadata/consts.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b2cfee9..6c220b5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ParentPath of json output for Exchange calendar now shows names instead of IDs. - Fixed failure when downloading huge amount of attachments +### Known Issues +- Restoring a OneDrive or SharePoint file with the same name as a file with that name as its M365 ID may restore both items. + ## [v0.6.1] (beta) - 2023-03-21 ### Added diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index d3ce9915c..43361ab2c 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -2,12 +2,14 @@ package testdata import ( "context" + "strings" "time" "github.com/alcionai/clues" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details/testdata" @@ -411,11 +413,13 @@ var ( }, }, { - Name: "NoSelectRepoItemName", - Expected: []details.DetailsEntry{}, + Name: "SelectRepoItemName", + Expected: []details.DetailsEntry{ + testdata.OneDriveItems[0], + }, Opts: utils.OneDriveOpts{ FileName: []string{ - testdata.OneDriveItemPath1.Item(), + strings.TrimSuffix(testdata.OneDriveItemPath1.Item(), metadata.DataFileSuffix), }, }, }, @@ -530,11 +534,13 @@ var ( }, }, { - Name: "NoSelectRepoItemName", - Expected: []details.DetailsEntry{}, + Name: "SelectRepoItemName", + Expected: []details.DetailsEntry{ + testdata.SharePointLibraryItems[0], + }, Opts: utils.SharePointOpts{ FileName: []string{ - testdata.SharePointLibraryItemPath1.Item(), + strings.TrimSuffix(testdata.SharePointLibraryItemPath1.Item(), metadata.DataFileSuffix), }, }, }, diff --git a/src/internal/connector/graph/metadata/metadata.go b/src/internal/connector/graph/metadata/metadata.go index b466ce185..cb08f7695 100644 --- a/src/internal/connector/graph/metadata/metadata.go +++ b/src/internal/connector/graph/metadata/metadata.go @@ -1,14 +1,14 @@ package metadata import ( - "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/path" ) func IsMetadataFile(p path.Path) bool { switch p.Service() { case path.OneDriveService: - return onedrive.IsMetaFile(p.Item()) + return metadata.HasMetaSuffix(p.Item()) default: return false diff --git a/src/internal/connector/graph/metadata/metadata_test.go b/src/internal/connector/graph/metadata/metadata_test.go index 41637b39e..94a2adc1d 100644 --- a/src/internal/connector/graph/metadata/metadata_test.go +++ b/src/internal/connector/graph/metadata/metadata_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/graph/metadata" - "github.com/alcionai/corso/src/internal/connector/onedrive" + odmetadata "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) @@ -29,12 +29,12 @@ var ( notMetaSuffixes = []string{ "", - onedrive.DataFileSuffix, + odmetadata.DataFileSuffix, } metaSuffixes = []string{ - onedrive.MetaFileSuffix, - onedrive.DirMetaFileSuffix, + odmetadata.MetaFileSuffix, + odmetadata.DirMetaFileSuffix, } cases = []testCase{ diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 16847f534..3a60bc701 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -730,7 +731,7 @@ func compareOneDriveItem( ) bool { // Skip OneDrive permissions in the folder that used to be the root. We don't // have a good way to materialize these in the test right now. - if rootDir && item.UUID() == onedrive.DirMetaFileSuffix { + if rootDir && item.UUID() == metadata.DirMetaFileSuffix { return false } @@ -742,8 +743,7 @@ func compareOneDriveItem( var ( displayName string name = item.UUID() - isMeta = strings.HasSuffix(name, onedrive.MetaFileSuffix) || - strings.HasSuffix(name, onedrive.DirMetaFileSuffix) + isMeta = metadata.HasMetaSuffix(name) ) if isMeta { @@ -780,7 +780,7 @@ func compareOneDriveItem( key := name - if strings.HasSuffix(name, onedrive.MetaFileSuffix) { + if strings.HasSuffix(name, metadata.MetaFileSuffix) { key = itemMeta.FileName } @@ -851,7 +851,7 @@ func compareOneDriveItem( // Display name in ItemInfo should match the name the file was given in the // test. Name used for the lookup key has a `.data` suffix to make it unique // from the metadata files' lookup keys. - assert.Equal(t, fileData.FileName, displayName+onedrive.DataFileSuffix) + assert.Equal(t, fileData.FileName, displayName+metadata.DataFileSuffix) return true } @@ -1225,8 +1225,7 @@ func collectionsForInfo( // We do not count metadata files against item count if backupVersion > 0 && (service == path.OneDriveService || service == path.SharePointService) && - (strings.HasSuffix(info.items[i].name, onedrive.MetaFileSuffix) || - strings.HasSuffix(info.items[i].name, onedrive.DirMetaFileSuffix)) { + metadata.HasMetaSuffix(info.items[i].name) { continue } diff --git a/src/internal/connector/graph_connector_onedrive_test.go b/src/internal/connector/graph_connector_onedrive_test.go index b6ca84956..d6447ae3f 100644 --- a/src/internal/connector/graph_connector_onedrive_test.go +++ b/src/internal/connector/graph_connector_onedrive_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" @@ -165,43 +166,43 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm permDat c.items = append(c.items, onedriveItemWithData( c.t, name, - name+onedrive.DataFileSuffix, + name+metadata.DataFileSuffix, fileData)) case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName: c.items = append(c.items, onedriveItemWithData( c.t, - name+onedrive.DataFileSuffix, - name+onedrive.DataFileSuffix, + name+metadata.DataFileSuffix, + name+metadata.DataFileSuffix, fileData)) - metadata := onedriveMetadata( + md := onedriveMetadata( c.t, "", - name+onedrive.MetaFileSuffix, - name+onedrive.MetaFileSuffix, + name+metadata.MetaFileSuffix, + name+metadata.MetaFileSuffix, perm, c.backupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, metadata) - c.aux = append(c.aux, metadata) + c.items = append(c.items, md) + c.aux = append(c.aux, md) case version.OneDrive6NameInMeta, version.OneDrive7LocationRef: c.items = append(c.items, onedriveItemWithData( c.t, - name+onedrive.DataFileSuffix, - name+onedrive.DataFileSuffix, + name+metadata.DataFileSuffix, + name+metadata.DataFileSuffix, fileData)) - metadata := onedriveMetadata( + md := onedriveMetadata( c.t, name, - name+onedrive.MetaFileSuffix, + name+metadata.MetaFileSuffix, name, perm, c.backupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, metadata) - c.aux = append(c.aux, metadata) + c.items = append(c.items, md) + c.aux = append(c.aux, md) default: assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion) @@ -222,8 +223,8 @@ func (c *onedriveCollection) withFolder(name string, perm permData) *onedriveCol onedriveMetadata( c.t, "", - name+onedrive.DirMetaFileSuffix, - name+onedrive.DirMetaFileSuffix, + name+metadata.DirMetaFileSuffix, + name+metadata.DirMetaFileSuffix, perm, c.backupVersion >= versionPermissionSwitchedToID)) @@ -255,16 +256,16 @@ func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection return c } - metadata := onedriveMetadata( + md := onedriveMetadata( c.t, name, - metaName+onedrive.DirMetaFileSuffix, - metaName+onedrive.DirMetaFileSuffix, + metaName+metadata.DirMetaFileSuffix, + metaName+metadata.DirMetaFileSuffix, perm, c.backupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, metadata) - c.aux = append(c.aux, metadata) + c.items = append(c.items, md) + c.aux = append(c.aux, md) return c } diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index e98920c87..1f9ce9c32 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -5,7 +5,6 @@ import ( "context" "io" "net/http" - "strings" "sync" "sync/atomic" "time" @@ -17,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive/api" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -39,18 +39,10 @@ const ( // be retried maxDownloadRetires = 3 - MetaFileSuffix = ".meta" - DirMetaFileSuffix = ".dirmeta" - DataFileSuffix = ".data" - // Used to compare in case of OneNote files MaxOneNoteFileSize = 2 * 1024 * 1024 * 1024 ) -func IsMetaFile(name string) bool { - return strings.HasSuffix(name, MetaFileSuffix) || strings.HasSuffix(name, DirMetaFileSuffix) -} - var ( _ data.BackupCollection = &Collection{} _ data.Stream = &Item{} @@ -524,12 +516,12 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { atomic.AddInt64(&itemsFound, 1) metaFileName = itemID - metaSuffix = MetaFileSuffix + metaSuffix = metadata.MetaFileSuffix } else { atomic.AddInt64(&dirsFound, 1) // metaFileName not set for directories so we get just ".dirmeta" - metaSuffix = DirMetaFileSuffix + metaSuffix = metadata.DirMetaFileSuffix } // Fetch metadata for the file @@ -556,7 +548,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { ctx = clues.Add(ctx, "backup_item_info", itemInfo) if isFile { - dataSuffix := DataFileSuffix + dataSuffix := metadata.DataFileSuffix // Construct a new lazy readCloser to feed to the collection consumer. // This ensures that downloads won't be attempted unless that consumer diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index 0f69b5455..b4328fe9b 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -266,7 +267,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { readItem := readItems[0] readItemInfo := readItem.(data.StreamInfo) - assert.Equal(t, testItemID+DataFileSuffix, readItem.UUID()) + assert.Equal(t, testItemID+metadata.DataFileSuffix, readItem.UUID()) require.Implements(t, (*data.StreamModTime)(nil), readItem) mt := readItem.(data.StreamModTime) @@ -292,7 +293,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { if test.source == OneDriveSource { readItemMeta := readItems[1] - assert.Equal(t, testItemID+MetaFileSuffix, readItemMeta.UUID()) + assert.Equal(t, testItemID+metadata.MetaFileSuffix, readItemMeta.UUID()) readMetaData, err := io.ReadAll(readItemMeta.ToReader()) require.NoError(t, err, clues.ToCore(err)) @@ -588,7 +589,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTim require.Equal(t, 1, collStatus.Metrics.Successes) for _, i := range readItems { - if strings.HasSuffix(i.UUID(), MetaFileSuffix) { + if strings.HasSuffix(i.UUID(), metadata.MetaFileSuffix) { content, err := io.ReadAll(i.ToReader()) require.NoError(t, err, clues.ToCore(err)) require.Equal(t, content, []byte("{}")) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index e7c1e782b..fdac083c8 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive/api" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -447,7 +448,7 @@ func (c *Collections) Get( } service, category := c.source.toPathServiceCat() - metadata, err := graph.MakeMetadataCollection( + md, err := graph.MakeMetadataCollection( c.tenant, c.resourceOwner, service, @@ -464,7 +465,7 @@ func (c *Collections) Get( // empty/missing and default to a full backup. logger.CtxErr(ctx, err).Info("making metadata collection for future incremental backups") } else { - collections = append(collections, metadata) + collections = append(collections, md) } // TODO(ashmrtn): Track and return the set of items to exclude. @@ -535,8 +536,8 @@ func (c *Collections) handleDelete( return nil } - excluded[itemID+DataFileSuffix] = struct{}{} - excluded[itemID+MetaFileSuffix] = struct{}{} + excluded[itemID+metadata.DataFileSuffix] = struct{}{} + excluded[itemID+metadata.MetaFileSuffix] = struct{}{} // Exchange counts items streamed through it which includes deletions so // add that here too. c.NumFiles++ @@ -853,8 +854,8 @@ func (c *Collections) UpdateCollections( // Always add a file to the excluded list. The file may have been // renamed/moved/modified, so we still have to drop the // original one and download a fresh copy. - excluded[itemID+DataFileSuffix] = struct{}{} - excluded[itemID+MetaFileSuffix] = struct{}{} + excluded[itemID+metadata.DataFileSuffix] = struct{}{} + excluded[itemID+metadata.MetaFileSuffix] = struct{}{} } default: diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index f2c6c1250..5598d701e 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -19,6 +19,7 @@ import ( gapi "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api/mock" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -147,8 +148,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestGetCanonicalPath() { func getDelList(files ...string) map[string]struct{} { delList := map[string]struct{}{} for _, file := range files { - delList[file+DataFileSuffix] = struct{}{} - delList[file+MetaFileSuffix] = struct{}{} + delList[file+metadata.DataFileSuffix] = struct{}{} + delList[file+metadata.MetaFileSuffix] = struct{}{} } return delList diff --git a/src/internal/connector/onedrive/metadata/consts.go b/src/internal/connector/onedrive/metadata/consts.go new file mode 100644 index 000000000..9ab8a6dcb --- /dev/null +++ b/src/internal/connector/onedrive/metadata/consts.go @@ -0,0 +1,13 @@ +package metadata + +import "strings" + +const ( + MetaFileSuffix = ".meta" + DirMetaFileSuffix = ".dirmeta" + DataFileSuffix = ".data" +) + +func HasMetaSuffix(name string) bool { + return strings.HasSuffix(name, MetaFileSuffix) || strings.HasSuffix(name, DirMetaFileSuffix) +} diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index 8c0fca2b7..44c4ae5f7 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" @@ -69,10 +70,10 @@ func getCollectionMetadata( // Root folder doesn't have a metadata file associated with it. folders := collectionPath.Folders() - metaName := folders[len(folders)-1] + DirMetaFileSuffix + metaName := folders[len(folders)-1] + metadata.DirMetaFileSuffix if backupVersion >= version.OneDrive5DirMetaNoName { - metaName = DirMetaFileSuffix + metaName = metadata.DirMetaFileSuffix } meta, err := fetchAndReadMetadata(ctx, dc, metaName) diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index ff3db56cb..d022d5d82 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -304,14 +305,14 @@ func restoreItem( // only v1+ backups from this point on - if strings.HasSuffix(itemUUID, MetaFileSuffix) { + if strings.HasSuffix(itemUUID, metadata.MetaFileSuffix) { // Just skip this for the moment since we moved the code to the above // item restore path. We haven't yet stopped fetching these items in // RestoreOp, so we still need to handle them in some way. return details.ItemInfo{}, true, nil } - if strings.HasSuffix(itemUUID, DirMetaFileSuffix) { + if strings.HasSuffix(itemUUID, metadata.DirMetaFileSuffix) { // Only the version.OneDrive1DataAndMetaFiles needed to deserialize the // permission for child folders here. Later versions can request // permissions inline when processing the collection. @@ -327,7 +328,7 @@ func restoreItem( return details.ItemInfo{}, true, clues.Wrap(err, "getting directory metadata").WithClues(ctx) } - trimmedPath := strings.TrimSuffix(itemPath.String(), DirMetaFileSuffix) + trimmedPath := strings.TrimSuffix(itemPath.String(), metadata.DirMetaFileSuffix) folderMetas[trimmedPath] = meta return details.ItemInfo{}, true, nil @@ -424,7 +425,7 @@ func restoreV1File( itemPath path.Path, itemData data.Stream, ) (details.ItemInfo, error) { - trimmedName := strings.TrimSuffix(itemData.UUID(), DataFileSuffix) + trimmedName := strings.TrimSuffix(itemData.UUID(), metadata.DataFileSuffix) itemID, itemInfo, err := restoreData( ctx, @@ -446,7 +447,7 @@ func restoreV1File( } // Fetch item permissions from the collection and restore them. - metaName := trimmedName + MetaFileSuffix + metaName := trimmedName + metadata.MetaFileSuffix meta, err := fetchAndReadMetadata(ctx, fetcher, metaName) if err != nil { @@ -485,10 +486,10 @@ func restoreV6File( itemPath path.Path, itemData data.Stream, ) (details.ItemInfo, error) { - trimmedName := strings.TrimSuffix(itemData.UUID(), DataFileSuffix) + trimmedName := strings.TrimSuffix(itemData.UUID(), metadata.DataFileSuffix) // Get metadata file so we can determine the file name. - metaName := trimmedName + MetaFileSuffix + metaName := trimmedName + metadata.MetaFileSuffix meta, err := fetchAndReadMetadata(ctx, fetcher, metaName) if err != nil { diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 507c4924c..7753bcd35 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -18,7 +18,7 @@ import ( "golang.org/x/exp/maps" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" - "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" @@ -390,8 +390,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { locPath, 3) mc.Names[0] = testFileName - mc.Names[1] = testFileName + onedrive.MetaFileSuffix - mc.Names[2] = storePath.Folders()[0] + onedrive.DirMetaFileSuffix + mc.Names[1] = testFileName + metadata.MetaFileSuffix + mc.Names[2] = storePath.Folders()[0] + metadata.DirMetaFileSuffix return []data.BackupCollection{mc} }, @@ -456,7 +456,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { continue } - assert.False(t, onedrive.IsMetaFile(entry.RepoRef), "metadata entry in details") + assert.False(t, metadata.HasMetaSuffix(entry.RepoRef), "metadata entry in details") } // Shouldn't have any items to merge because the cached files are metadata diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 2fe7a43aa..b18488d3f 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -11,7 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/connector" - "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/kopia" @@ -471,7 +471,7 @@ func getBackupDetails( if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker { for _, d := range deets.Entries { if d.OneDrive != nil { - d.OneDrive.IsMeta = onedrive.IsMetaFile(d.RepoRef) + d.OneDrive.IsMeta = metadata.HasMetaSuffix(d.RepoRef) } } } diff --git a/src/pkg/selectors/onedrive.go b/src/pkg/selectors/onedrive.go index ef2ca489f..63dba94b5 100644 --- a/src/pkg/selectors/onedrive.go +++ b/src/pkg/selectors/onedrive.go @@ -3,10 +3,12 @@ package selectors import ( "context" "fmt" + "strings" "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" @@ -400,9 +402,11 @@ func (c oneDriveCategory) pathValues( // Ignore `drives//root:` for folder comparison rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String() + itemID := strings.TrimSuffix(repo.Item(), metadata.DataFileSuffix) + result := map[categorizer][]string{ OneDriveFolder: {rFld}, - OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef}, + OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef, itemID}, } if len(ent.LocationRef) > 0 { diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index 8dcc861e8..055fcb0d2 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -270,7 +270,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() { expected := map[categorizer][]string{ OneDriveFolder: {"dir1/dir2"}, - OneDriveItem: {fileName, shortRef}, + OneDriveItem: {fileName, shortRef, fileName + "-id"}, } ent := details.DetailsEntry{ diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 7250efc65..b5ac5bcdc 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -3,10 +3,12 @@ package selectors import ( "context" "fmt" + "strings" "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" @@ -522,6 +524,7 @@ func (c sharePointCategory) pathValues( folderCat, itemCat categorizer itemName = repo.Item() dropDriveFolderPrefix bool + itemID string ) switch c { @@ -532,6 +535,7 @@ func (c sharePointCategory) pathValues( dropDriveFolderPrefix = true folderCat, itemCat = SharePointLibraryFolder, SharePointLibraryItem + itemID = strings.TrimSuffix(itemName, metadata.DataFileSuffix) itemName = ent.SharePoint.ItemName case SharePointList, SharePointListItem: @@ -555,6 +559,10 @@ func (c sharePointCategory) pathValues( itemCat: {itemName, ent.ShortRef}, } + if len(itemID) > 0 { + result[itemCat] = append(result[itemCat], itemID) + } + if len(ent.LocationRef) > 0 { result[folderCat] = append(result[folderCat], ent.LocationRef) } diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 5e91a617d..1f3544b71 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -353,7 +353,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { pathElems: driveElems, expected: map[categorizer][]string{ SharePointLibraryFolder: {"dir1/dir2"}, - SharePointLibraryItem: {itemName, shortRef}, + SharePointLibraryItem: {itemName, shortRef, itemName + "-id"}, }, }, {