diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 3699eda50..c3233e231 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -190,6 +190,9 @@ func handleDeleteCmd(cmd *cobra.Command, args []string) error { // common handlers // --------------------------------------------------------------------------- +// standard set of selector behavior that we want used in the cli +var defaultSelectorConfig = selectors.Config{OnlyMatchItemNames: true} + func runBackups( ctx context.Context, r repository.Repository, @@ -203,6 +206,8 @@ func runBackups( ) for _, discSel := range selectorSet { + discSel.Configure(defaultSelectorConfig) + var ( owner = discSel.DiscreteOwner ictx = clues.Add(ctx, "resource_owner", owner) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 5b5e48f75..e5bcecf96 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -317,6 +317,7 @@ func runDetailsExchangeCmd( if !skipReduce { sel := utils.IncludeExchangeRestoreDataSelectors(opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) utils.FilterExchangeRestoreInfoSelectors(sel, opts) d = sel.Reduce(ctx, d, errs) } diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index ae2c8a1ed..006ae087b 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -278,6 +278,7 @@ func runDetailsOneDriveCmd( if !skipReduce { sel := utils.IncludeOneDriveRestoreDataSelectors(opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) utils.FilterOneDriveRestoreInfoSelectors(sel, opts) d = sel.Reduce(ctx, d, errs) } diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index 4778bf387..bf3ff3c71 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -362,6 +362,7 @@ func runDetailsSharePointCmd( if !skipReduce { sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) utils.FilterSharePointRestoreInfoSelectors(sel, opts) d = sel.Reduce(ctx, d, errs) } diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index 43361ab2c..9511f58ab 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -2,14 +2,12 @@ package testdata import ( "context" - "strings" "time" "github.com/alcionai/clues" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details/testdata" @@ -201,10 +199,10 @@ var ( }, }, { - Name: "MailID", + Name: "MailItemRef", Expected: []details.DetailsEntry{testdata.ExchangeEmailItems[0]}, Opts: utils.ExchangeOpts{ - Email: []string{testdata.ExchangeEmailItemPath1.Item()}, + Email: []string{testdata.ExchangeEmailItems[0].ItemRef}, }, }, { @@ -413,13 +411,11 @@ var ( }, }, { - Name: "SelectRepoItemName", - Expected: []details.DetailsEntry{ - testdata.OneDriveItems[0], - }, + Name: "ItemRefMatchesNothing", + Expected: []details.DetailsEntry{}, Opts: utils.OneDriveOpts{ FileName: []string{ - strings.TrimSuffix(testdata.OneDriveItemPath1.Item(), metadata.DataFileSuffix), + testdata.OneDriveItems[0].ItemRef, }, }, }, @@ -534,13 +530,11 @@ var ( }, }, { - Name: "SelectRepoItemName", - Expected: []details.DetailsEntry{ - testdata.SharePointLibraryItems[0], - }, + Name: "ItemRefMatchesNothing", + Expected: []details.DetailsEntry{}, Opts: utils.SharePointOpts{ FileName: []string{ - strings.TrimSuffix(testdata.SharePointLibraryItemPath1.Item(), metadata.DataFileSuffix), + testdata.SharePointLibraryItems[0].ItemRef, }, }, }, diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index b12f579f8..1e1f85e96 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -903,8 +903,7 @@ func traverseBaseDir( oldDirPath, currentPath, dEntry, - roots, - ) + roots) }) if err != nil { return clues.Wrap(err, "traversing base directory") diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index 64579e017..157e8b80c 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -481,11 +481,17 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { require.Len(t, cp.pending, len(ci)) + foundItems := map[string]bool{} + for k, v := range ci { if cachedTest.cached { cp.CachedFile(k, v.totalBytes) } + if v.info != nil && v.info.repoPath != nil { + foundItems[v.info.repoPath.Item()] = false + } + cp.FinishedFile(k, v.err) } @@ -496,6 +502,14 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { for _, entry := range entries { assert.Equal(t, !cachedTest.cached, entry.Updated) + + foundItems[entry.ItemRef] = true + } + + if test.expectedNumEntries > 0 { + for item, found := range foundItems { + assert.Truef(t, found, "details missing item: %s", item) + } } }) } diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 698e5f118..3e9e36805 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -320,6 +320,7 @@ func makeDetailsEntry( RepoRef: p.String(), ShortRef: p.ShortRef(), ParentRef: p.ToBuilder().Dir().ShortRef(), + ItemRef: p.Item(), LocationRef: lr, ItemInfo: details.ItemInfo{}, Updated: updated, diff --git a/src/internal/streamstore/collectables_test.go b/src/internal/streamstore/collectables_test.go index 8eb3dc7fd..8c146e3a6 100644 --- a/src/internal/streamstore/collectables_test.go +++ b/src/internal/streamstore/collectables_test.go @@ -191,6 +191,7 @@ func (suite *StreamStoreIntgSuite) TestStreamer() { assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef) assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef) assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef) + assert.Equal(t, deets.Entries[0].ItemRef, readDeets.Entries[0].ItemRef) assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated) assert.NotNil(t, readDeets.Entries[0].Exchange) assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange) diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index 0e99583f2..32074c9c6 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "strconv" + "strings" "sync" "time" @@ -14,6 +15,7 @@ import ( "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" ) @@ -388,6 +390,7 @@ func (d *Details) add( ShortRef: repoRef.ShortRef(), ParentRef: repoRef.ToBuilder().Dir().ShortRef(), LocationRef: locationRef.String(), + ItemRef: repoRef.Item(), Updated: updated, ItemInfo: info, } @@ -418,6 +421,9 @@ func (d *Details) add( elements := repoRef.Elements() elements = append(elements[:len(elements)-1], filename, repoRef.Item()) entry.ShortRef = path.Builder{}.Append(elements...).ShortRef() + + // clean metadata suffixes from item refs + entry.ItemRef = withoutMetadataSuffix(entry.ItemRef) } d.Entries = append(d.Entries, entry) @@ -437,6 +443,16 @@ func UnmarshalTo(d *Details) func(io.ReadCloser) error { } } +// remove metadata file suffixes from the string. +// assumes only one suffix is applied to any given id. +func withoutMetadataSuffix(id string) string { + id = strings.TrimSuffix(id, metadata.DirMetaFileSuffix) + id = strings.TrimSuffix(id, metadata.MetaFileSuffix) + id = strings.TrimSuffix(id, metadata.DataFileSuffix) + + return id +} + // -------------------------------------------------------------------------------- // Entry // -------------------------------------------------------------------------------- @@ -458,6 +474,12 @@ type DetailsEntry struct { // Currently only implemented for Exchange Calendars. LocationRef string `json:"locationRef,omitempty"` + // ItemRef contains the stable id of the item itself. ItemRef is not + // guaranteed to be unique within a repository. Uniqueness guarantees + // maximally inherit from the source item. Eg: Entries for m365 mail items + // are only as unique as m365 mail item IDs themselves. + ItemRef string `json:"itemRef,omitempty"` + // Indicates the item was added or updated in this backup // Always `true` for full backups Updated bool `json:"updated"` diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index b0c421954..eba378f1b 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "strings" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" @@ -48,6 +50,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { RepoRef: "reporef", ShortRef: "deadbeef", LocationRef: "locationref", + ItemRef: "itemref", }, expectHs: []string{"ID"}, expectVs: []string{"deadbeef"}, @@ -58,6 +61,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { RepoRef: "reporef", ShortRef: "deadbeef", LocationRef: "locationref", + ItemRef: "itemref", ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: ExchangeEvent, @@ -78,6 +82,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { RepoRef: "reporef", ShortRef: "deadbeef", LocationRef: "locationref", + ItemRef: "itemref", ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: ExchangeContact, @@ -94,6 +99,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { RepoRef: "reporef", ShortRef: "deadbeef", LocationRef: "locationref", + ItemRef: "itemref", ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: ExchangeMail, @@ -114,6 +120,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { RepoRef: "reporef", ShortRef: "deadbeef", LocationRef: "locationref", + ItemRef: "itemref", ItemInfo: ItemInfo{ SharePoint: &SharePointInfo{ ItemName: "itemName", @@ -145,6 +152,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { RepoRef: "reporef", ShortRef: "deadbeef", LocationRef: "locationref", + ItemRef: "itemref", ItemInfo: ItemInfo{ OneDrive: &OneDriveInfo{ ItemName: "itemName", @@ -187,6 +195,7 @@ func exchangeEntry(t *testing.T, id string, size int, it ItemType) DetailsEntry ShortRef: rr.ShortRef(), ParentRef: rr.ToBuilder().Dir().ShortRef(), LocationRef: rr.Folder(true), + ItemRef: rr.Item(), ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: it, @@ -248,11 +257,14 @@ func oneDriveishEntry(t *testing.T, id string, size int, it ItemType) DetailsEnt ShortRef: rr.ShortRef(), ParentRef: rr.ToBuilder().Dir().ShortRef(), LocationRef: loc.String(), + ItemRef: rr.Item(), ItemInfo: info, } } func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() { + itemID := "foo" + t := suite.T() table := []struct { name string @@ -266,23 +278,23 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() { }{ { name: "Exchange Email", - entry: exchangeEntry(t, "foo", 42, ExchangeMail), + entry: exchangeEntry(t, itemID, 42, ExchangeMail), shortRefEqual: assert.Equal, }, { name: "OneDrive File", - entry: oneDriveishEntry(t, "foo", 42, OneDriveItem), + entry: oneDriveishEntry(t, itemID, 42, OneDriveItem), shortRefEqual: assert.NotEqual, }, { name: "SharePoint File", - entry: oneDriveishEntry(t, "foo", 42, SharePointLibrary), + entry: oneDriveishEntry(t, itemID, 42, SharePointLibrary), shortRefEqual: assert.NotEqual, }, { name: "Legacy SharePoint File", entry: func() DetailsEntry { - res := oneDriveishEntry(t, "foo", 42, SharePointLibrary) + res := oneDriveishEntry(t, itemID, 42, SharePointLibrary) res.SharePoint.ItemType = OneDriveItem return res @@ -734,6 +746,7 @@ var pathItemsTable = []struct { { RepoRef: "abcde", LocationRef: "locationref", + ItemRef: "itemref", }, }, expectRepoRefs: []string{"abcde"}, @@ -745,10 +758,12 @@ var pathItemsTable = []struct { { RepoRef: "abcde", LocationRef: "locationref", + ItemRef: "itemref", }, { RepoRef: "12345", LocationRef: "locationref2", + ItemRef: "itemref2", }, }, expectRepoRefs: []string{"abcde", "12345"}, @@ -760,10 +775,12 @@ var pathItemsTable = []struct { { RepoRef: "abcde", LocationRef: "locationref", + ItemRef: "itemref", }, { RepoRef: "12345", LocationRef: "locationref2", + ItemRef: "itemref2", }, { RepoRef: "deadbeef", @@ -788,6 +805,7 @@ var pathItemsTable = []struct { { RepoRef: "foo.meta", LocationRef: "locationref.dirmeta", + ItemRef: "itemref.meta", ItemInfo: ItemInfo{ OneDrive: &OneDriveInfo{IsMeta: false}, }, @@ -795,6 +813,7 @@ var pathItemsTable = []struct { { RepoRef: "is-meta-file", LocationRef: "locationref-meta-file", + ItemRef: "itemref-meta-file", ItemInfo: ItemInfo{ OneDrive: &OneDriveInfo{IsMeta: true}, }, @@ -809,14 +828,17 @@ var pathItemsTable = []struct { { RepoRef: "abcde", LocationRef: "locationref", + ItemRef: "itemref", }, { RepoRef: "12345", LocationRef: "locationref2", + ItemRef: "itemref2", }, { RepoRef: "foo.meta", LocationRef: "locationref.dirmeta", + ItemRef: "itemref.dirmeta", ItemInfo: ItemInfo{ OneDrive: &OneDriveInfo{IsMeta: false}, }, @@ -824,6 +846,7 @@ var pathItemsTable = []struct { { RepoRef: "is-meta-file", LocationRef: "locationref-meta-file", + ItemRef: "itemref-meta-file", ItemInfo: ItemInfo{ OneDrive: &OneDriveInfo{IsMeta: true}, }, @@ -831,6 +854,7 @@ var pathItemsTable = []struct { { RepoRef: "deadbeef", LocationRef: "locationref3", + ItemRef: "itemref3", ItemInfo: ItemInfo{ Folder: &FolderInfo{ DisplayName: "test folder", @@ -912,7 +936,7 @@ func (suite *DetailsUnitSuite) TestDetailsModel_FilterMetaFiles() { assert.Len(t, d.Entries, 3) } -func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() { +func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() { t := suite.T() b := Builder{} @@ -937,8 +961,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() { "root:", "folder", name + "-id", - }, - ) + }) otherItemPath := makeItemPath( t, @@ -952,8 +975,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() { "folder", name + "-id", name, - }, - ) + }) err := b.Add( itemPath, @@ -961,7 +983,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() { &path.Builder{}, false, info) - require.NoError(t, err) + require.NoError(t, err, clues.ToCore(err)) items := b.Details().Items() require.Len(t, items, 1) @@ -971,6 +993,45 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() { assert.NotEqual(t, otherItemPath.ShortRef(), items[0].ShortRef, "same ShortRef as subfolder item") } +func (suite *DetailsUnitSuite) TestBuilder_Add_cleansFileIDSuffixes() { + var ( + t = suite.T() + b = Builder{} + svc = path.OneDriveService + cat = path.FilesCategory + info = ItemInfo{ + OneDrive: &OneDriveInfo{ + ItemType: OneDriveItem, + ItemName: "in", + DriveName: "dn", + DriveID: "d", + }, + } + + dataSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.DataFileSuffix}) + dirMetaSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.DirMetaFileSuffix}) + metaSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.MetaFileSuffix}) + ) + + // Don't need to generate folders for this entry, we just want the itemRef + loc := &path.Builder{} + + err := b.Add(dataSfx, loc, false, info) + require.NoError(t, err, clues.ToCore(err)) + + err = b.Add(dirMetaSfx, loc, false, info) + require.NoError(t, err, clues.ToCore(err)) + + err = b.Add(metaSfx, loc, false, info) + require.NoError(t, err, clues.ToCore(err)) + + for _, ent := range b.Details().Items() { + assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.DirMetaFileSuffix)) + assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.MetaFileSuffix)) + assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.DataFileSuffix)) + } +} + func makeItemPath( t *testing.T, service path.ServiceType, diff --git a/src/pkg/backup/details/testdata/testdata.go b/src/pkg/backup/details/testdata/testdata.go index 763080c41..d51937abd 100644 --- a/src/pkg/backup/details/testdata/testdata.go +++ b/src/pkg/backup/details/testdata/testdata.go @@ -60,6 +60,7 @@ var ( RepoRef: ExchangeEmailItemPath1.String(), ShortRef: ExchangeEmailItemPath1.ShortRef(), ParentRef: ExchangeEmailItemPath1.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath1.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -73,6 +74,7 @@ var ( RepoRef: ExchangeEmailItemPath2.String(), ShortRef: ExchangeEmailItemPath2.ShortRef(), ParentRef: ExchangeEmailItemPath2.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath2.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -86,6 +88,7 @@ var ( RepoRef: ExchangeEmailItemPath3.String(), ShortRef: ExchangeEmailItemPath3.ShortRef(), ParentRef: ExchangeEmailItemPath3.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath3.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -108,6 +111,7 @@ var ( RepoRef: ExchangeContactsItemPath1.String(), ShortRef: ExchangeContactsItemPath1.ShortRef(), ParentRef: ExchangeContactsItemPath1.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath1.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeContact, @@ -119,6 +123,7 @@ var ( RepoRef: ExchangeContactsItemPath2.String(), ShortRef: ExchangeContactsItemPath2.ShortRef(), ParentRef: ExchangeContactsItemPath2.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath2.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeContact, @@ -139,6 +144,7 @@ var ( RepoRef: ExchangeEventsItemPath1.String(), ShortRef: ExchangeEventsItemPath1.ShortRef(), ParentRef: ExchangeEventsItemPath1.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath2.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeEvent, @@ -153,6 +159,7 @@ var ( RepoRef: ExchangeEventsItemPath2.String(), ShortRef: ExchangeEventsItemPath2.ShortRef(), ParentRef: ExchangeEventsItemPath2.ToBuilder().Dir().ShortRef(), + ItemRef: ExchangeEmailItemPath2.Item(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeEvent, @@ -183,6 +190,7 @@ var ( RepoRef: OneDriveItemPath1.String(), ShortRef: OneDriveItemPath1.ShortRef(), ParentRef: OneDriveItemPath1.ToBuilder().Dir().ShortRef(), + ItemRef: OneDriveItemPath1.Item(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -199,6 +207,7 @@ var ( RepoRef: OneDriveItemPath2.String(), ShortRef: OneDriveItemPath2.ShortRef(), ParentRef: OneDriveItemPath2.ToBuilder().Dir().ShortRef(), + ItemRef: OneDriveItemPath2.Item(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -215,6 +224,7 @@ var ( RepoRef: OneDriveItemPath3.String(), ShortRef: OneDriveItemPath3.ShortRef(), ParentRef: OneDriveItemPath3.ToBuilder().Dir().ShortRef(), + ItemRef: OneDriveItemPath3.Item(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -247,6 +257,7 @@ var ( RepoRef: SharePointLibraryItemPath1.String(), ShortRef: SharePointLibraryItemPath1.ShortRef(), ParentRef: SharePointLibraryItemPath1.ToBuilder().Dir().ShortRef(), + ItemRef: SharePointLibraryItemPath1.Item(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -263,6 +274,7 @@ var ( RepoRef: SharePointLibraryItemPath2.String(), ShortRef: SharePointLibraryItemPath2.ShortRef(), ParentRef: SharePointLibraryItemPath2.ToBuilder().Dir().ShortRef(), + ItemRef: SharePointLibraryItemPath2.Item(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -279,6 +291,7 @@ var ( RepoRef: SharePointLibraryItemPath3.String(), ShortRef: SharePointLibraryItemPath3.ShortRef(), ParentRef: SharePointLibraryItemPath3.ToBuilder().Dir().ShortRef(), + ItemRef: SharePointLibraryItemPath3.Item(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, diff --git a/src/pkg/selectors/example_selectors_test.go b/src/pkg/selectors/example_selectors_test.go index ff556a306..2e3260748 100644 --- a/src/pkg/selectors/example_selectors_test.go +++ b/src/pkg/selectors/example_selectors_test.go @@ -125,6 +125,7 @@ var ( { RepoRef: "tID/exchange/your-user-id/email/example/itemID", ShortRef: "xyz", + ItemRef: "123", ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index 63001f94f..ccee7f948 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -594,6 +594,7 @@ func (ec exchangeCategory) isLeaf() bool { func (ec exchangeCategory) pathValues( repo path.Path, ent details.DetailsEntry, + cfg Config, ) (map[categorizer][]string, error) { var folderCat, itemCat categorizer @@ -611,9 +612,14 @@ func (ec exchangeCategory) pathValues( return nil, clues.New("bad exchanageCategory").With("category", ec) } + item := ent.ItemRef + if len(item) == 0 { + item = repo.Item() + } + result := map[categorizer][]string{ folderCat: {repo.Folder(false)}, - itemCat: {repo.Item(), ent.ShortRef}, + itemCat: {item, ent.ShortRef}, } if len(ent.LocationRef) > 0 { diff --git a/src/pkg/selectors/exchange_test.go b/src/pkg/selectors/exchange_test.go index a7416894f..703ea6f72 100644 --- a/src/pkg/selectors/exchange_test.go +++ b/src/pkg/selectors/exchange_test.go @@ -728,6 +728,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() { ent = details.DetailsEntry{ RepoRef: repo.String(), ShortRef: short, + ItemRef: mail, LocationRef: loc, } ) @@ -769,7 +770,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() { scopes := setScopesToDefault(test.scope) var aMatch bool for _, scope := range scopes { - pvs, err := ExchangeMail.pathValues(repo, ent) + pvs, err := ExchangeMail.pathValues(repo, ent, Config{}) require.NoError(t, err) if matchesPathValues(scope, ExchangeMail, pvs) { @@ -1312,14 +1313,17 @@ func (suite *ExchangeSelectorSuite) TestScopesByCategory() { } func (suite *ExchangeSelectorSuite) TestPasses() { - short := "thisisahashofsomekind" - entry := details.DetailsEntry{ShortRef: short} - const ( mid = "mailID" cat = ExchangeMail ) + short := "thisisahashofsomekind" + entry := details.DetailsEntry{ + ShortRef: short, + ItemRef: mid, + } + var ( es = NewExchangeRestore(Any()) otherMail = setScopesToDefault(es.Mails(Any(), []string{"smarf"})) @@ -1352,7 +1356,7 @@ func (suite *ExchangeSelectorSuite) TestPasses() { suite.Run(test.name, func() { t := suite.T() - pvs, err := cat.pathValues(repo, ent) + pvs, err := cat.pathValues(repo, ent, Config{}) require.NoError(t, err) result := passes( @@ -1493,9 +1497,10 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() { ent := details.DetailsEntry{ RepoRef: test.path.String(), ShortRef: "short", + ItemRef: test.path.Item(), } - pvs, err := test.cat.pathValues(test.path, ent) + pvs, err := test.cat.pathValues(test.path, ent, Config{}) require.NoError(t, err) assert.Equal(t, test.expect, pvs) }) diff --git a/src/pkg/selectors/helpers_test.go b/src/pkg/selectors/helpers_test.go index 70ff9d281..e85aa8d86 100644 --- a/src/pkg/selectors/helpers_test.go +++ b/src/pkg/selectors/helpers_test.go @@ -60,6 +60,7 @@ func (mc mockCategorizer) isLeaf() bool { func (mc mockCategorizer) pathValues( repo path.Path, ent details.DetailsEntry, + cfg Config, ) (map[categorizer][]string, error) { return map[categorizer][]string{ rootCatStub: {"root"}, diff --git a/src/pkg/selectors/onedrive.go b/src/pkg/selectors/onedrive.go index 63dba94b5..bd1837feb 100644 --- a/src/pkg/selectors/onedrive.go +++ b/src/pkg/selectors/onedrive.go @@ -3,12 +3,10 @@ package selectors import ( "context" "fmt" - "strings" "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" @@ -394,6 +392,7 @@ func (c oneDriveCategory) isLeaf() bool { func (c oneDriveCategory) pathValues( repo path.Path, ent details.DetailsEntry, + cfg Config, ) (map[categorizer][]string, error) { if ent.OneDrive == nil { return nil, clues.New("no OneDrive ItemInfo in details") @@ -402,11 +401,18 @@ func (c oneDriveCategory) pathValues( // Ignore `drives//root:` for folder comparison rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String() - itemID := strings.TrimSuffix(repo.Item(), metadata.DataFileSuffix) + item := ent.ItemRef + if len(item) == 0 { + item = repo.Item() + } + + if cfg.OnlyMatchItemNames { + item = ent.ItemInfo.OneDrive.ItemName + } result := map[categorizer][]string{ OneDriveFolder: {rFld}, - OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef, itemID}, + OneDriveItem: {item, ent.ShortRef}, } if len(ent.LocationRef) > 0 { diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index 055fcb0d2..3bf953bf9 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -173,6 +173,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() { Entries: []details.DetailsEntry{ { RepoRef: file, + ItemRef: "file", ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -182,6 +183,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() { }, { RepoRef: file2, + ItemRef: "file2", ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -191,6 +193,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() { }, { RepoRef: file3, + // item ref intentionally blank to assert fallback case ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -211,36 +214,69 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() { deets *details.Details makeSelector func() *OneDriveRestore expect []string + cfg Config }{ { - "all", - deets, - func() *OneDriveRestore { + name: "all", + deets: deets, + makeSelector: func() *OneDriveRestore { odr := NewOneDriveRestore(Any()) odr.Include(odr.AllData()) return odr }, - arr(file, file2, file3), + expect: arr(file, file2, file3), }, { - "only match file", - deets, - func() *OneDriveRestore { + name: "only match file", + deets: deets, + makeSelector: func() *OneDriveRestore { + odr := NewOneDriveRestore(Any()) + odr.Include(odr.Items(Any(), []string{"file2"})) + return odr + }, + expect: arr(file2), + }, + { + name: "id doesn't match name", + deets: deets, + makeSelector: func() *OneDriveRestore { + odr := NewOneDriveRestore(Any()) + odr.Include(odr.Items(Any(), []string{"file2"})) + return odr + }, + expect: []string{}, + cfg: Config{OnlyMatchItemNames: true}, + }, + { + name: "only match file name", + deets: deets, + makeSelector: func() *OneDriveRestore { odr := NewOneDriveRestore(Any()) odr.Include(odr.Items(Any(), []string{"fileName2"})) return odr }, - arr(file2), + expect: arr(file2), + cfg: Config{OnlyMatchItemNames: true}, }, { - "only match folder", - deets, - func() *OneDriveRestore { + name: "name doesn't match id", + deets: deets, + makeSelector: func() *OneDriveRestore { + odr := NewOneDriveRestore(Any()) + odr.Include(odr.Items(Any(), []string{"fileName2"})) + return odr + }, + expect: []string{}, + }, + { + name: "only match folder", + deets: deets, + makeSelector: func() *OneDriveRestore { odr := NewOneDriveRestore([]string{"uid"}) odr.Include(odr.Folders([]string{"folderA/folderB", "folderA/folderC"})) return odr }, - arr(file, file2), + expect: arr(file, file2), }, } for _, test := range table { @@ -251,6 +287,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() { defer flush() sel := test.makeSelector() + sel.Configure(test.cfg) results := sel.Reduce(ctx, test.deets, fault.New(true)) paths := results.Paths() assert.Equal(t, test.expect, paths) @@ -262,30 +299,68 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() { t := suite.T() fileName := "file" + fileID := fileName + "-id" shortRef := "short" - elems := []string{"drive", "driveID", "root:", "dir1", "dir2", fileName + "-id"} + elems := []string{"drive", "driveID", "root:", "dir1", "dir2", fileID} filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...) require.NoError(t, err, clues.ToCore(err)) - expected := map[categorizer][]string{ - OneDriveFolder: {"dir1/dir2"}, - OneDriveItem: {fileName, shortRef, fileName + "-id"}, - } - - ent := details.DetailsEntry{ - RepoRef: filePath.String(), - ShortRef: shortRef, - ItemInfo: details.ItemInfo{ - OneDrive: &details.OneDriveInfo{ - ItemName: fileName, + table := []struct { + name string + pathElems []string + expected map[categorizer][]string + cfg Config + }{ + { + name: "items", + pathElems: elems, + expected: map[categorizer][]string{ + OneDriveFolder: {"dir1/dir2"}, + OneDriveItem: {fileID, shortRef}, }, + cfg: Config{}, + }, + { + name: "items w/ name", + pathElems: elems, + expected: map[categorizer][]string{ + OneDriveFolder: {"dir1/dir2"}, + OneDriveItem: {fileName, shortRef}, + }, + cfg: Config{OnlyMatchItemNames: true}, }, } - r, err := OneDriveItem.pathValues(filePath, ent) - require.NoError(t, err) - assert.Equal(t, expected, r) + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + itemPath, err := path.Build( + "tenant", + "site", + path.OneDriveService, + path.FilesCategory, + true, + test.pathElems...) + require.NoError(t, err, clues.ToCore(err)) + + ent := details.DetailsEntry{ + RepoRef: filePath.String(), + ShortRef: shortRef, + ItemRef: fileID, + ItemInfo: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + ItemName: fileName, + }, + }, + } + + pv, err := OneDriveItem.pathValues(itemPath, ent, test.cfg) + require.NoError(t, err) + assert.Equal(t, test.expected, pv) + }) + } } func (suite *OneDriveSelectorSuite) TestOneDriveScope_MatchesInfo() { diff --git a/src/pkg/selectors/scopes.go b/src/pkg/selectors/scopes.go index 1842fa021..51519546f 100644 --- a/src/pkg/selectors/scopes.go +++ b/src/pkg/selectors/scopes.go @@ -89,7 +89,7 @@ type ( // folderCat: folder, // itemCat: itemID, // } - pathValues(path.Path, details.DetailsEntry) (map[categorizer][]string, error) + pathValues(path.Path, details.DetailsEntry, Config) (map[categorizer][]string, error) // pathKeys produces a list of categorizers that can be used as keys in the pathValues // map. The combination of the two funcs generically interprets the context of the @@ -389,7 +389,7 @@ func reduce[T scopeT, C categoryT]( continue } - pv, err := dc.pathValues(repoPath, *ent) + pv, err := dc.pathValues(repoPath, *ent, s.Cfg) if err != nil { el.AddRecoverable(clues.Wrap(err, "getting path values").WithClues(ictx)) continue diff --git a/src/pkg/selectors/scopes_test.go b/src/pkg/selectors/scopes_test.go index 0fc33791d..ed4020f42 100644 --- a/src/pkg/selectors/scopes_test.go +++ b/src/pkg/selectors/scopes_test.go @@ -366,7 +366,7 @@ func (suite *SelectorScopesSuite) TestPasses() { } ) - pvs, err := cat.pathValues(pth, entry) + pvs, err := cat.pathValues(pth, entry, Config{}) require.NoError(suite.T(), err) for _, test := range reduceTestTable { diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 1800d3a44..02dd4427f 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -122,6 +122,16 @@ type Selector struct { // A slice of inclusion scopes. Comparators must match either one of these, // or all filters, to be included. Includes []scope `json:"includes,omitempty"` + + Cfg Config `json:"cfg,omitempty"` +} + +// Config defines broad-scale selector behavior. +type Config struct { + // OnlyMatchItemNames tells the reducer to ignore matching on itemRef values + // and other item IDs in favor of matching the item name. Normal behavior only + // matches on itemRefs. + OnlyMatchItemNames bool } // helper for specific selector instance constructors. @@ -140,6 +150,11 @@ func newSelector(s service, resourceOwners []string) Selector { } } +// Configure sets the selector configuration. +func (s *Selector) Configure(cfg Config) { + s.Cfg = cfg +} + // DiscreteResourceOwners returns the list of individual resourceOwners used // in the selector. // TODO(rkeepers): remove in favor of split and s.DiscreteOwner diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index b5ac5bcdc..defa6d206 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -3,12 +3,10 @@ package selectors import ( "context" "fmt" - "strings" "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" @@ -519,10 +517,10 @@ func (c sharePointCategory) isLeaf() bool { func (c sharePointCategory) pathValues( repo path.Path, ent details.DetailsEntry, + cfg Config, ) (map[categorizer][]string, error) { var ( folderCat, itemCat categorizer - itemName = repo.Item() dropDriveFolderPrefix bool itemID string ) @@ -535,8 +533,6 @@ func (c sharePointCategory) pathValues( dropDriveFolderPrefix = true folderCat, itemCat = SharePointLibraryFolder, SharePointLibraryItem - itemID = strings.TrimSuffix(itemName, metadata.DataFileSuffix) - itemName = ent.SharePoint.ItemName case SharePointList, SharePointListItem: folderCat, itemCat = SharePointList, SharePointListItem @@ -554,9 +550,18 @@ func (c sharePointCategory) pathValues( rFld = path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String() } + item := ent.ItemRef + if len(item) == 0 { + item = repo.Item() + } + + if cfg.OnlyMatchItemNames { + item = ent.ItemInfo.SharePoint.ItemName + } + result := map[categorizer][]string{ folderCat: {rFld}, - itemCat: {itemName, ent.ShortRef}, + itemCat: {item, ent.ShortRef}, } if len(itemID) > 0 { diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 1f3544b71..e606ff5e2 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -220,6 +220,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { Entries: []details.DetailsEntry{ { RepoRef: item, + ItemRef: "item", ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -229,6 +230,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { }, { RepoRef: item2, + // ItemRef intentionally blank to test fallback case ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -238,6 +240,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { }, { RepoRef: item3, + ItemRef: "item3", ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -247,6 +250,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { }, { RepoRef: item4, + ItemRef: "item4", ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointPage, @@ -256,6 +260,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { }, { RepoRef: item5, + // ItemRef intentionally blank to test fallback case ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointPage, @@ -276,6 +281,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets *details.Details makeSelector func() *SharePointRestore expect []string + cfg Config }{ { name: "all", @@ -290,12 +296,44 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { { name: "only match item", deets: deets, + makeSelector: func() *SharePointRestore { + odr := NewSharePointRestore(Any()) + odr.Include(odr.LibraryItems(Any(), []string{"item2"})) + return odr + }, + expect: arr(item2), + }, + { + name: "id doesn't match name", + deets: deets, + makeSelector: func() *SharePointRestore { + odr := NewSharePointRestore(Any()) + odr.Include(odr.LibraryItems(Any(), []string{"item2"})) + return odr + }, + expect: []string{}, + cfg: Config{OnlyMatchItemNames: true}, + }, + { + name: "only match item name", + deets: deets, makeSelector: func() *SharePointRestore { odr := NewSharePointRestore(Any()) odr.Include(odr.LibraryItems(Any(), []string{"itemName2"})) return odr }, expect: arr(item2), + cfg: Config{OnlyMatchItemNames: true}, + }, + { + name: "name doesn't match", + deets: deets, + makeSelector: func() *SharePointRestore { + odr := NewSharePointRestore(Any()) + odr.Include(odr.LibraryItems(Any(), []string{"itemName2"})) + return odr + }, + expect: []string{}, }, { name: "only match folder", @@ -326,6 +364,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { defer flush() sel := test.makeSelector() + sel.Configure(test.cfg) results := sel.Reduce(ctx, test.deets, fault.New(true)) paths := results.Paths() assert.Equal(t, test.expect, paths) @@ -336,9 +375,10 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { var ( itemName = "item" + itemID = "item-id" shortRef = "short" - driveElems = []string{"drive", "drive!id", "root:", "dir1", "dir2", itemName + "-id"} - elems = []string{"dir1", "dir2", itemName + "-id"} + driveElems = []string{"drive", "drive!id", "root:", "dir1", "dir2", itemID} + elems = []string{"dir1", "dir2", itemID} ) table := []struct { @@ -346,6 +386,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { sc sharePointCategory pathElems []string expected map[categorizer][]string + cfg Config }{ { name: "SharePoint Libraries", @@ -353,8 +394,19 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { pathElems: driveElems, expected: map[categorizer][]string{ SharePointLibraryFolder: {"dir1/dir2"}, - SharePointLibraryItem: {itemName, shortRef, itemName + "-id"}, + SharePointLibraryItem: {itemID, shortRef}, }, + cfg: Config{}, + }, + { + name: "SharePoint Libraries w/ name", + sc: SharePointLibraryItem, + pathElems: driveElems, + expected: map[categorizer][]string{ + SharePointLibraryFolder: {"dir1/dir2"}, + SharePointLibraryItem: {itemName, shortRef}, + }, + cfg: Config{OnlyMatchItemNames: true}, }, { name: "SharePoint Lists", @@ -362,8 +414,9 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { pathElems: elems, expected: map[categorizer][]string{ SharePointList: {"dir1/dir2"}, - SharePointListItem: {"item-id", shortRef}, + SharePointListItem: {itemID, shortRef}, }, + cfg: Config{}, }, } @@ -383,6 +436,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { ent := details.DetailsEntry{ RepoRef: itemPath.String(), ShortRef: shortRef, + ItemRef: itemPath.Item(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemName: itemName, @@ -390,7 +444,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { }, } - pv, err := test.sc.pathValues(itemPath, ent) + pv, err := test.sc.pathValues(itemPath, ent, test.cfg) require.NoError(t, err) assert.Equal(t, test.expected, pv) })