diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b2cfee9..6c220b5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ParentPath of json output for Exchange calendar now shows names instead of IDs. - Fixed failure when downloading huge amount of attachments +### Known Issues +- Restoring a OneDrive or SharePoint file with the same name as a file with that name as its M365 ID may restore both items. + ## [v0.6.1] (beta) - 2023-03-21 ### Added diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index d3ce9915c..43361ab2c 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -2,12 +2,14 @@ package testdata import ( "context" + "strings" "time" "github.com/alcionai/clues" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details/testdata" @@ -411,11 +413,13 @@ var ( }, }, { - Name: "NoSelectRepoItemName", - Expected: []details.DetailsEntry{}, + Name: "SelectRepoItemName", + Expected: []details.DetailsEntry{ + testdata.OneDriveItems[0], + }, Opts: utils.OneDriveOpts{ FileName: []string{ - testdata.OneDriveItemPath1.Item(), + strings.TrimSuffix(testdata.OneDriveItemPath1.Item(), metadata.DataFileSuffix), }, }, }, @@ -530,11 +534,13 @@ var ( }, }, { - Name: "NoSelectRepoItemName", - Expected: []details.DetailsEntry{}, + Name: "SelectRepoItemName", + Expected: []details.DetailsEntry{ + testdata.SharePointLibraryItems[0], + }, Opts: utils.SharePointOpts{ FileName: []string{ - testdata.SharePointLibraryItemPath1.Item(), + strings.TrimSuffix(testdata.SharePointLibraryItemPath1.Item(), metadata.DataFileSuffix), }, }, }, diff --git a/src/internal/connector/graph/metadata/metadata.go b/src/internal/connector/graph/metadata/metadata.go index b466ce185..cb08f7695 100644 --- a/src/internal/connector/graph/metadata/metadata.go +++ b/src/internal/connector/graph/metadata/metadata.go @@ -1,14 +1,14 @@ package metadata import ( - "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/path" ) func IsMetadataFile(p path.Path) bool { switch p.Service() { case path.OneDriveService: - return onedrive.IsMetaFile(p.Item()) + return metadata.HasMetaSuffix(p.Item()) default: return false diff --git a/src/internal/connector/graph/metadata/metadata_test.go b/src/internal/connector/graph/metadata/metadata_test.go index 41637b39e..94a2adc1d 100644 --- a/src/internal/connector/graph/metadata/metadata_test.go +++ b/src/internal/connector/graph/metadata/metadata_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/graph/metadata" - "github.com/alcionai/corso/src/internal/connector/onedrive" + odmetadata "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) @@ -29,12 +29,12 @@ var ( notMetaSuffixes = []string{ "", - onedrive.DataFileSuffix, + odmetadata.DataFileSuffix, } metaSuffixes = []string{ - onedrive.MetaFileSuffix, - onedrive.DirMetaFileSuffix, + odmetadata.MetaFileSuffix, + odmetadata.DirMetaFileSuffix, } cases = []testCase{ diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 16847f534..3a60bc701 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -730,7 +731,7 @@ func compareOneDriveItem( ) bool { // Skip OneDrive permissions in the folder that used to be the root. We don't // have a good way to materialize these in the test right now. - if rootDir && item.UUID() == onedrive.DirMetaFileSuffix { + if rootDir && item.UUID() == metadata.DirMetaFileSuffix { return false } @@ -742,8 +743,7 @@ func compareOneDriveItem( var ( displayName string name = item.UUID() - isMeta = strings.HasSuffix(name, onedrive.MetaFileSuffix) || - strings.HasSuffix(name, onedrive.DirMetaFileSuffix) + isMeta = metadata.HasMetaSuffix(name) ) if isMeta { @@ -780,7 +780,7 @@ func compareOneDriveItem( key := name - if strings.HasSuffix(name, onedrive.MetaFileSuffix) { + if strings.HasSuffix(name, metadata.MetaFileSuffix) { key = itemMeta.FileName } @@ -851,7 +851,7 @@ func compareOneDriveItem( // Display name in ItemInfo should match the name the file was given in the // test. Name used for the lookup key has a `.data` suffix to make it unique // from the metadata files' lookup keys. - assert.Equal(t, fileData.FileName, displayName+onedrive.DataFileSuffix) + assert.Equal(t, fileData.FileName, displayName+metadata.DataFileSuffix) return true } @@ -1225,8 +1225,7 @@ func collectionsForInfo( // We do not count metadata files against item count if backupVersion > 0 && (service == path.OneDriveService || service == path.SharePointService) && - (strings.HasSuffix(info.items[i].name, onedrive.MetaFileSuffix) || - strings.HasSuffix(info.items[i].name, onedrive.DirMetaFileSuffix)) { + metadata.HasMetaSuffix(info.items[i].name) { continue } diff --git a/src/internal/connector/graph_connector_onedrive_test.go b/src/internal/connector/graph_connector_onedrive_test.go index b6ca84956..d6447ae3f 100644 --- a/src/internal/connector/graph_connector_onedrive_test.go +++ b/src/internal/connector/graph_connector_onedrive_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" @@ -165,43 +166,43 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm permDat c.items = append(c.items, onedriveItemWithData( c.t, name, - name+onedrive.DataFileSuffix, + name+metadata.DataFileSuffix, fileData)) case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName: c.items = append(c.items, onedriveItemWithData( c.t, - name+onedrive.DataFileSuffix, - name+onedrive.DataFileSuffix, + name+metadata.DataFileSuffix, + name+metadata.DataFileSuffix, fileData)) - metadata := onedriveMetadata( + md := onedriveMetadata( c.t, "", - name+onedrive.MetaFileSuffix, - name+onedrive.MetaFileSuffix, + name+metadata.MetaFileSuffix, + name+metadata.MetaFileSuffix, perm, c.backupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, metadata) - c.aux = append(c.aux, metadata) + c.items = append(c.items, md) + c.aux = append(c.aux, md) case version.OneDrive6NameInMeta, version.OneDrive7LocationRef: c.items = append(c.items, onedriveItemWithData( c.t, - name+onedrive.DataFileSuffix, - name+onedrive.DataFileSuffix, + name+metadata.DataFileSuffix, + name+metadata.DataFileSuffix, fileData)) - metadata := onedriveMetadata( + md := onedriveMetadata( c.t, name, - name+onedrive.MetaFileSuffix, + name+metadata.MetaFileSuffix, name, perm, c.backupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, metadata) - c.aux = append(c.aux, metadata) + c.items = append(c.items, md) + c.aux = append(c.aux, md) default: assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion) @@ -222,8 +223,8 @@ func (c *onedriveCollection) withFolder(name string, perm permData) *onedriveCol onedriveMetadata( c.t, "", - name+onedrive.DirMetaFileSuffix, - name+onedrive.DirMetaFileSuffix, + name+metadata.DirMetaFileSuffix, + name+metadata.DirMetaFileSuffix, perm, c.backupVersion >= versionPermissionSwitchedToID)) @@ -255,16 +256,16 @@ func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection return c } - metadata := onedriveMetadata( + md := onedriveMetadata( c.t, name, - metaName+onedrive.DirMetaFileSuffix, - metaName+onedrive.DirMetaFileSuffix, + metaName+metadata.DirMetaFileSuffix, + metaName+metadata.DirMetaFileSuffix, perm, c.backupVersion >= versionPermissionSwitchedToID) - c.items = append(c.items, metadata) - c.aux = append(c.aux, metadata) + c.items = append(c.items, md) + c.aux = append(c.aux, md) return c } diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index e98920c87..1f9ce9c32 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -5,7 +5,6 @@ import ( "context" "io" "net/http" - "strings" "sync" "sync/atomic" "time" @@ -17,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive/api" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -39,18 +39,10 @@ const ( // be retried maxDownloadRetires = 3 - MetaFileSuffix = ".meta" - DirMetaFileSuffix = ".dirmeta" - DataFileSuffix = ".data" - // Used to compare in case of OneNote files MaxOneNoteFileSize = 2 * 1024 * 1024 * 1024 ) -func IsMetaFile(name string) bool { - return strings.HasSuffix(name, MetaFileSuffix) || strings.HasSuffix(name, DirMetaFileSuffix) -} - var ( _ data.BackupCollection = &Collection{} _ data.Stream = &Item{} @@ -524,12 +516,12 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { atomic.AddInt64(&itemsFound, 1) metaFileName = itemID - metaSuffix = MetaFileSuffix + metaSuffix = metadata.MetaFileSuffix } else { atomic.AddInt64(&dirsFound, 1) // metaFileName not set for directories so we get just ".dirmeta" - metaSuffix = DirMetaFileSuffix + metaSuffix = metadata.DirMetaFileSuffix } // Fetch metadata for the file @@ -556,7 +548,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { ctx = clues.Add(ctx, "backup_item_info", itemInfo) if isFile { - dataSuffix := DataFileSuffix + dataSuffix := metadata.DataFileSuffix // Construct a new lazy readCloser to feed to the collection consumer. // This ensures that downloads won't be attempted unless that consumer diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index 0f69b5455..b4328fe9b 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -266,7 +267,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { readItem := readItems[0] readItemInfo := readItem.(data.StreamInfo) - assert.Equal(t, testItemID+DataFileSuffix, readItem.UUID()) + assert.Equal(t, testItemID+metadata.DataFileSuffix, readItem.UUID()) require.Implements(t, (*data.StreamModTime)(nil), readItem) mt := readItem.(data.StreamModTime) @@ -292,7 +293,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { if test.source == OneDriveSource { readItemMeta := readItems[1] - assert.Equal(t, testItemID+MetaFileSuffix, readItemMeta.UUID()) + assert.Equal(t, testItemID+metadata.MetaFileSuffix, readItemMeta.UUID()) readMetaData, err := io.ReadAll(readItemMeta.ToReader()) require.NoError(t, err, clues.ToCore(err)) @@ -588,7 +589,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTim require.Equal(t, 1, collStatus.Metrics.Successes) for _, i := range readItems { - if strings.HasSuffix(i.UUID(), MetaFileSuffix) { + if strings.HasSuffix(i.UUID(), metadata.MetaFileSuffix) { content, err := io.ReadAll(i.ToReader()) require.NoError(t, err, clues.ToCore(err)) require.Equal(t, content, []byte("{}")) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index e7c1e782b..fdac083c8 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive/api" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -447,7 +448,7 @@ func (c *Collections) Get( } service, category := c.source.toPathServiceCat() - metadata, err := graph.MakeMetadataCollection( + md, err := graph.MakeMetadataCollection( c.tenant, c.resourceOwner, service, @@ -464,7 +465,7 @@ func (c *Collections) Get( // empty/missing and default to a full backup. logger.CtxErr(ctx, err).Info("making metadata collection for future incremental backups") } else { - collections = append(collections, metadata) + collections = append(collections, md) } // TODO(ashmrtn): Track and return the set of items to exclude. @@ -535,8 +536,8 @@ func (c *Collections) handleDelete( return nil } - excluded[itemID+DataFileSuffix] = struct{}{} - excluded[itemID+MetaFileSuffix] = struct{}{} + excluded[itemID+metadata.DataFileSuffix] = struct{}{} + excluded[itemID+metadata.MetaFileSuffix] = struct{}{} // Exchange counts items streamed through it which includes deletions so // add that here too. c.NumFiles++ @@ -853,8 +854,8 @@ func (c *Collections) UpdateCollections( // Always add a file to the excluded list. The file may have been // renamed/moved/modified, so we still have to drop the // original one and download a fresh copy. - excluded[itemID+DataFileSuffix] = struct{}{} - excluded[itemID+MetaFileSuffix] = struct{}{} + excluded[itemID+metadata.DataFileSuffix] = struct{}{} + excluded[itemID+metadata.MetaFileSuffix] = struct{}{} } default: diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index f2c6c1250..5598d701e 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -19,6 +19,7 @@ import ( gapi "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api/mock" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -147,8 +148,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestGetCanonicalPath() { func getDelList(files ...string) map[string]struct{} { delList := map[string]struct{}{} for _, file := range files { - delList[file+DataFileSuffix] = struct{}{} - delList[file+MetaFileSuffix] = struct{}{} + delList[file+metadata.DataFileSuffix] = struct{}{} + delList[file+metadata.MetaFileSuffix] = struct{}{} } return delList diff --git a/src/internal/connector/onedrive/metadata/consts.go b/src/internal/connector/onedrive/metadata/consts.go new file mode 100644 index 000000000..9ab8a6dcb --- /dev/null +++ b/src/internal/connector/onedrive/metadata/consts.go @@ -0,0 +1,13 @@ +package metadata + +import "strings" + +const ( + MetaFileSuffix = ".meta" + DirMetaFileSuffix = ".dirmeta" + DataFileSuffix = ".data" +) + +func HasMetaSuffix(name string) bool { + return strings.HasSuffix(name, MetaFileSuffix) || strings.HasSuffix(name, DirMetaFileSuffix) +} diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index 8c0fca2b7..44c4ae5f7 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" @@ -69,10 +70,10 @@ func getCollectionMetadata( // Root folder doesn't have a metadata file associated with it. folders := collectionPath.Folders() - metaName := folders[len(folders)-1] + DirMetaFileSuffix + metaName := folders[len(folders)-1] + metadata.DirMetaFileSuffix if backupVersion >= version.OneDrive5DirMetaNoName { - metaName = DirMetaFileSuffix + metaName = metadata.DirMetaFileSuffix } meta, err := fetchAndReadMetadata(ctx, dc, metaName) diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index ff3db56cb..d022d5d82 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -304,14 +305,14 @@ func restoreItem( // only v1+ backups from this point on - if strings.HasSuffix(itemUUID, MetaFileSuffix) { + if strings.HasSuffix(itemUUID, metadata.MetaFileSuffix) { // Just skip this for the moment since we moved the code to the above // item restore path. We haven't yet stopped fetching these items in // RestoreOp, so we still need to handle them in some way. return details.ItemInfo{}, true, nil } - if strings.HasSuffix(itemUUID, DirMetaFileSuffix) { + if strings.HasSuffix(itemUUID, metadata.DirMetaFileSuffix) { // Only the version.OneDrive1DataAndMetaFiles needed to deserialize the // permission for child folders here. Later versions can request // permissions inline when processing the collection. @@ -327,7 +328,7 @@ func restoreItem( return details.ItemInfo{}, true, clues.Wrap(err, "getting directory metadata").WithClues(ctx) } - trimmedPath := strings.TrimSuffix(itemPath.String(), DirMetaFileSuffix) + trimmedPath := strings.TrimSuffix(itemPath.String(), metadata.DirMetaFileSuffix) folderMetas[trimmedPath] = meta return details.ItemInfo{}, true, nil @@ -424,7 +425,7 @@ func restoreV1File( itemPath path.Path, itemData data.Stream, ) (details.ItemInfo, error) { - trimmedName := strings.TrimSuffix(itemData.UUID(), DataFileSuffix) + trimmedName := strings.TrimSuffix(itemData.UUID(), metadata.DataFileSuffix) itemID, itemInfo, err := restoreData( ctx, @@ -446,7 +447,7 @@ func restoreV1File( } // Fetch item permissions from the collection and restore them. - metaName := trimmedName + MetaFileSuffix + metaName := trimmedName + metadata.MetaFileSuffix meta, err := fetchAndReadMetadata(ctx, fetcher, metaName) if err != nil { @@ -485,10 +486,10 @@ func restoreV6File( itemPath path.Path, itemData data.Stream, ) (details.ItemInfo, error) { - trimmedName := strings.TrimSuffix(itemData.UUID(), DataFileSuffix) + trimmedName := strings.TrimSuffix(itemData.UUID(), metadata.DataFileSuffix) // Get metadata file so we can determine the file name. - metaName := trimmedName + MetaFileSuffix + metaName := trimmedName + metadata.MetaFileSuffix meta, err := fetchAndReadMetadata(ctx, fetcher, metaName) if err != nil { diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 507c4924c..7753bcd35 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -18,7 +18,7 @@ import ( "golang.org/x/exp/maps" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" - "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" @@ -390,8 +390,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { locPath, 3) mc.Names[0] = testFileName - mc.Names[1] = testFileName + onedrive.MetaFileSuffix - mc.Names[2] = storePath.Folders()[0] + onedrive.DirMetaFileSuffix + mc.Names[1] = testFileName + metadata.MetaFileSuffix + mc.Names[2] = storePath.Folders()[0] + metadata.DirMetaFileSuffix return []data.BackupCollection{mc} }, @@ -456,7 +456,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { continue } - assert.False(t, onedrive.IsMetaFile(entry.RepoRef), "metadata entry in details") + assert.False(t, metadata.HasMetaSuffix(entry.RepoRef), "metadata entry in details") } // Shouldn't have any items to merge because the cached files are metadata diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 2fe7a43aa..b18488d3f 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -11,7 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/connector" - "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/kopia" @@ -471,7 +471,7 @@ func getBackupDetails( if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker { for _, d := range deets.Entries { if d.OneDrive != nil { - d.OneDrive.IsMeta = onedrive.IsMetaFile(d.RepoRef) + d.OneDrive.IsMeta = metadata.HasMetaSuffix(d.RepoRef) } } } diff --git a/src/pkg/selectors/onedrive.go b/src/pkg/selectors/onedrive.go index ef2ca489f..63dba94b5 100644 --- a/src/pkg/selectors/onedrive.go +++ b/src/pkg/selectors/onedrive.go @@ -3,10 +3,12 @@ package selectors import ( "context" "fmt" + "strings" "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" @@ -400,9 +402,11 @@ func (c oneDriveCategory) pathValues( // Ignore `drives//root:` for folder comparison rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String() + itemID := strings.TrimSuffix(repo.Item(), metadata.DataFileSuffix) + result := map[categorizer][]string{ OneDriveFolder: {rFld}, - OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef}, + OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef, itemID}, } if len(ent.LocationRef) > 0 { diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index 8dcc861e8..055fcb0d2 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -270,7 +270,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() { expected := map[categorizer][]string{ OneDriveFolder: {"dir1/dir2"}, - OneDriveItem: {fileName, shortRef}, + OneDriveItem: {fileName, shortRef, fileName + "-id"}, } ent := details.DetailsEntry{ diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 7250efc65..b5ac5bcdc 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -3,10 +3,12 @@ package selectors import ( "context" "fmt" + "strings" "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" @@ -522,6 +524,7 @@ func (c sharePointCategory) pathValues( folderCat, itemCat categorizer itemName = repo.Item() dropDriveFolderPrefix bool + itemID string ) switch c { @@ -532,6 +535,7 @@ func (c sharePointCategory) pathValues( dropDriveFolderPrefix = true folderCat, itemCat = SharePointLibraryFolder, SharePointLibraryItem + itemID = strings.TrimSuffix(itemName, metadata.DataFileSuffix) itemName = ent.SharePoint.ItemName case SharePointList, SharePointListItem: @@ -555,6 +559,10 @@ func (c sharePointCategory) pathValues( itemCat: {itemName, ent.ShortRef}, } + if len(itemID) > 0 { + result[itemCat] = append(result[itemCat], itemID) + } + if len(ent.LocationRef) > 0 { result[folderCat] = append(result[folderCat], ent.LocationRef) } diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 5e91a617d..1f3544b71 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -353,7 +353,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { pathElems: driveElems, expected: map[categorizer][]string{ SharePointLibraryFolder: {"dir1/dir2"}, - SharePointLibraryItem: {itemName, shortRef}, + SharePointLibraryItem: {itemName, shortRef, itemName + "-id"}, }, }, {