From 320726342535db374a4a5909e304fee66dc06496 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Tue, 21 Feb 2023 16:04:22 +0530 Subject: [PATCH] Add both data and meta file to exclude list (#2557) ## Description This makes sure both the `.data` and `.meta` files for a OneDrive item are added to the exclude list so that both are deleted. It does nothing for dirs as of now as we will be moving the directory metadata to within the directory so that we can leverage kopia tree operations to take care of that. ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * https://github.com/alcionai/corso/issues/2117 ## Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/onedrive/collections.go | 10 +++- .../connector/onedrive/collections_test.go | 56 ++++++++++--------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 130339b9a..2a1707a67 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -584,6 +584,13 @@ func (c *Collections) UpdateCollections( // the deleted folder/package. delete(newPaths, *item.GetId()) + // TODO(meain): Directory metadata files should be + // moved into the directory instead of having a + // `.dirmeta` file at the same level as the + // directory. This way we can make sure it is moved + // and deleted along with the directory and don't have + // to be handled separately. + if prevPath == nil { // It is possible that an item was created and // deleted between two delta invocations. In @@ -661,7 +668,8 @@ func (c *Collections) UpdateCollections( // deleted, we want to avoid it. If it was // renamed/moved/modified, we still have to drop the // original one and download a fresh copy. - excluded[*item.GetId()] = struct{}{} + excluded[*item.GetId()+DataFileSuffix] = struct{}{} + excluded[*item.GetId()+MetaFileSuffix] = struct{}{} } if item.GetDeleted() != nil { diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 7e5ec8688..d4465ba62 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -137,6 +137,16 @@ func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() { } } +func getDelList(files ...string) map[string]struct{} { + delList := map[string]struct{}{} + for _, file := range files { + delList[file+DataFileSuffix] = struct{}{} + delList[file+MetaFileSuffix] = struct{}{} + } + + return delList +} + func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any())[0] @@ -198,7 +208,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedMetadataPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{"file": {}}, + expectedExcludes: getDelList("file"), }, { testCase: "Single Folder", @@ -266,7 +276,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "folder": expectedPath("/folder"), "package": expectedPath("/package"), }, - expectedExcludes: map[string]struct{}{"fileInRoot": {}, "fileInFolder": {}, "fileInPackage": {}}, + expectedExcludes: getDelList("fileInRoot", "fileInFolder", "fileInPackage"), }, { testCase: "contains folder selector", @@ -299,7 +309,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "subfolder": expectedPath("/folder/subfolder"), "folder2": expectedPath("/folder/subfolder/folder"), }, - expectedExcludes: map[string]struct{}{"fileInFolder": {}, "fileInFolder2": {}}, + expectedExcludes: getDelList("fileInFolder", "fileInFolder2"), }, { testCase: "prefix subfolder selector", @@ -329,7 +339,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "root": expectedPath(""), "folder2": expectedPath("/folder/subfolder/folder"), }, - expectedExcludes: map[string]struct{}{"fileInFolder2": {}}, + expectedExcludes: getDelList("fileInFolder2"), }, { testCase: "match subfolder selector", @@ -356,7 +366,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedMetadataPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{"fileInSubfolder": {}}, + expectedExcludes: getDelList("fileInSubfolder"), }, { testCase: "not moved folder tree", @@ -433,7 +443,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "root": expectedPath(""), "folder": expectedPath("/folder"), }, - expectedExcludes: map[string]struct{}{"file": {}}, + expectedExcludes: getDelList("file"), }, { testCase: "moved folder tree with file no previous", @@ -457,7 +467,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "root": expectedPath(""), "folder": expectedPath("/folder2"), }, - expectedExcludes: map[string]struct{}{"file": {}}, + expectedExcludes: getDelList("file"), }, { testCase: "moved folder tree with file no previous 1", @@ -480,7 +490,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "root": expectedPath(""), "folder": expectedPath("/folder"), }, - expectedExcludes: map[string]struct{}{"file": {}}, + expectedExcludes: getDelList("file"), }, { testCase: "moved folder tree and subfolder 1", @@ -577,7 +587,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "folder2": expectedPath("/folder2"), "subfolder": expectedPath("/folder/subfolder"), }, - expectedExcludes: map[string]struct{}{"itemInSubfolder": {}, "itemInFolder2": {}}, + expectedExcludes: getDelList("itemInSubfolder", "itemInFolder2"), }, { testCase: "moved folder tree multiple times", @@ -605,9 +615,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { "folder": expectedPath("/folder2"), "subfolder": expectedPath("/folder2/subfolder"), }, - expectedExcludes: map[string]struct{}{ - "file": {}, - }, + expectedExcludes: getDelList("file"), }, { testCase: "deleted folder and package", @@ -699,9 +707,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedMetadataPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{ - "item": {}, - }, + expectedExcludes: getDelList("item"), }, } @@ -1237,9 +1243,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { expectedFolderPaths: map[string]map[string]string{ driveID1: {"root": rootFolderPath1}, }, - expectedDelList: map[string]struct{}{ - "file": {}, - }, + expectedDelList: getDelList("file"), }, { name: "OneDrive_OneItemPage_NoFolders_NoErrors", @@ -1268,7 +1272,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { expectedFolderPaths: map[string]map[string]string{ driveID1: {"root": rootFolderPath1}, }, - expectedDelList: map[string]struct{}{"file": {}}, + expectedDelList: getDelList("file"), }, { name: "OneDrive_OneItemPage_NoErrors", @@ -1302,7 +1306,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { "folder": folderPath1, }, }, - expectedDelList: map[string]struct{}{"file": {}}, + expectedDelList: getDelList("file"), }, { name: "OneDrive_OneItemPage_NoErrors_FileRenamedMultiple", @@ -1337,7 +1341,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { "folder": folderPath1, }, }, - expectedDelList: map[string]struct{}{"file": {}}, + expectedDelList: getDelList("file"), }, { name: "OneDrive_OneItemPage_NoErrors_FileMovedMultiple", @@ -1371,7 +1375,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { "folder": folderPath1, }, }, - expectedDelList: map[string]struct{}{"file": {}}, + expectedDelList: getDelList("file"), }, { name: "OneDrive_OneItemPage_EmptyDelta_NoErrors", @@ -1398,7 +1402,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { }, expectedDeltaURLs: map[string]string{}, expectedFolderPaths: map[string]map[string]string{}, - expectedDelList: map[string]struct{}{"file": {}}, + expectedDelList: getDelList("file"), }, { name: "OneDrive_TwoItemPages_NoErrors", @@ -1440,7 +1444,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { "folder": folderPath1, }, }, - expectedDelList: map[string]struct{}{"file": {}, "file2": {}}, + expectedDelList: getDelList("file", "file2"), }, { name: "TwoDrives_OneItemPageEach_NoErrors", @@ -1495,7 +1499,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { "folder2": folderPath2, }, }, - expectedDelList: map[string]struct{}{"file": {}, "file2": {}}, + expectedDelList: getDelList("file", "file2"), }, { name: "OneDrive_OneItemPage_Errors", @@ -1629,7 +1633,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() { "folder": folderPath1, }, }, - expectedDelList: map[string]struct{}{"file": {}, "file2": {}}, + expectedDelList: getDelList("file", "file2"), doNotMergeItems: false, }, {