larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add itemref to details entry (#3160)

Browse Source 
Adds an new details entry field: itemRef.
This holds a stable, semi-unique identifier to the item represented by that entry.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3027

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-04-19 17:25:11 -06:00 committed by GitHub
parent 2121b40293
commit 306db14339
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 359 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/cli/backup/backup.go
View File

@ -190,6 +190,9 @@ func handleDeleteCmd(cmd *cobra.Command, args []string) error {
// common handlers
// ---------------------------------------------------------------------------
// standard set of selector behavior that we want used in the cli
var defaultSelectorConfig = selectors.Config{OnlyMatchItemNames: true}
func runBackups(
ctx context.Context,
r repository.Repository,
@ -203,6 +206,8 @@ func runBackups(
)
for _, discSel := range selectorSet {
discSel.Configure(defaultSelectorConfig)
var (
owner = discSel.DiscreteOwner
ictx = clues.Add(ctx, "resource_owner", owner)

1
src/cli/backup/exchange.go
View File

@ -317,6 +317,7 @@ func runDetailsExchangeCmd(
if !skipReduce {
sel := utils.IncludeExchangeRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterExchangeRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}

1
src/cli/backup/onedrive.go
View File

@ -278,6 +278,7 @@ func runDetailsOneDriveCmd(
if !skipReduce {
sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}

1
src/cli/backup/sharepoint.go
View File

@ -362,6 +362,7 @@ func runDetailsSharePointCmd(
if !skipReduce {
sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterSharePointRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}

22
src/cli/utils/testdata/opts.go vendored
View File

@ -2,14 +2,12 @@ package testdata
import (
"context"
"strings"
"time"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/details/testdata"
@ -201,10 +199,10 @@ var (
},
},
{
Name: "MailID",
Name: "MailItemRef",
Expected: []details.DetailsEntry{testdata.ExchangeEmailItems[0]},
Opts: utils.ExchangeOpts{
Email: []string{testdata.ExchangeEmailItemPath1.Item()},
Email: []string{testdata.ExchangeEmailItems[0].ItemRef},
},
},
{
@ -413,13 +411,11 @@ var (
},
},
{
Name: "SelectRepoItemName",
Expected: []details.DetailsEntry{
testdata.OneDriveItems[0],
},
Name: "ItemRefMatchesNothing",
Expected: []details.DetailsEntry{},
Opts: utils.OneDriveOpts{
FileName: []string{
strings.TrimSuffix(testdata.OneDriveItemPath1.Item(), metadata.DataFileSuffix),
testdata.OneDriveItems[0].ItemRef,
},
},
},
@ -534,13 +530,11 @@ var (
},
},
{
Name: "SelectRepoItemName",
Expected: []details.DetailsEntry{
testdata.SharePointLibraryItems[0],
},
Name: "ItemRefMatchesNothing",
Expected: []details.DetailsEntry{},
Opts: utils.SharePointOpts{
FileName: []string{
strings.TrimSuffix(testdata.SharePointLibraryItemPath1.Item(), metadata.DataFileSuffix),
testdata.SharePointLibraryItems[0].ItemRef,
},
},
},

3
src/internal/kopia/upload.go
View File

@ -903,8 +903,7 @@ func traverseBaseDir(
oldDirPath,
currentPath,
dEntry,
roots,
)
roots)
})
if err != nil {
return clues.Wrap(err, "traversing base directory")

14
src/internal/kopia/upload_test.go
View File

@ -481,11 +481,17 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
require.Len(t, cp.pending, len(ci))
foundItems := map[string]bool{}
for k, v := range ci {
if cachedTest.cached {
cp.CachedFile(k, v.totalBytes)
}
if v.info != nil && v.info.repoPath != nil {
foundItems[v.info.repoPath.Item()] = false
}
cp.FinishedFile(k, v.err)
}
@ -496,6 +502,14 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
for _, entry := range entries {
assert.Equal(t, !cachedTest.cached, entry.Updated)
foundItems[entry.ItemRef] = true
}
if test.expectedNumEntries > 0 {
for item, found := range foundItems {
assert.Truef(t, found, "details missing item: %s", item)
}
}
})
}

1
src/internal/operations/backup_test.go
View File

@ -320,6 +320,7 @@ func makeDetailsEntry(
RepoRef: p.String(),
ShortRef: p.ShortRef(),
ParentRef: p.ToBuilder().Dir().ShortRef(),
ItemRef: p.Item(),
LocationRef: lr,
ItemInfo: details.ItemInfo{},
Updated: updated,

1
src/internal/streamstore/collectables_test.go
View File

@ -191,6 +191,7 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef)
assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef)
assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef)
assert.Equal(t, deets.Entries[0].ItemRef, readDeets.Entries[0].ItemRef)
assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated)
assert.NotNil(t, readDeets.Entries[0].Exchange)
assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange)

22
src/pkg/backup/details/details.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"io"
"strconv"
"strings"
"sync"
"time"
@ -14,6 +15,7 @@ import (
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path"
)
@ -388,6 +390,7 @@ func (d *Details) add(
ShortRef: repoRef.ShortRef(),
ParentRef: repoRef.ToBuilder().Dir().ShortRef(),
LocationRef: locationRef.String(),
ItemRef: repoRef.Item(),
Updated: updated,
ItemInfo: info,
}
@ -418,6 +421,9 @@ func (d *Details) add(
elements := repoRef.Elements()
elements = append(elements[:len(elements)-1], filename, repoRef.Item())
entry.ShortRef = path.Builder{}.Append(elements...).ShortRef()
// clean metadata suffixes from item refs
entry.ItemRef = withoutMetadataSuffix(entry.ItemRef)
}
d.Entries = append(d.Entries, entry)
@ -437,6 +443,16 @@ func UnmarshalTo(d *Details) func(io.ReadCloser) error {
}
}
// remove metadata file suffixes from the string.
// assumes only one suffix is applied to any given id.
func withoutMetadataSuffix(id string) string {
id = strings.TrimSuffix(id, metadata.DirMetaFileSuffix)
id = strings.TrimSuffix(id, metadata.MetaFileSuffix)
id = strings.TrimSuffix(id, metadata.DataFileSuffix)
return id
}
// --------------------------------------------------------------------------------
// Entry
// --------------------------------------------------------------------------------
@ -458,6 +474,12 @@ type DetailsEntry struct {
// Currently only implemented for Exchange Calendars.
LocationRef string `json:"locationRef,omitempty"`
// ItemRef contains the stable id of the item itself. ItemRef is not
// guaranteed to be unique within a repository. Uniqueness guarantees
// maximally inherit from the source item. Eg: Entries for m365 mail items
// are only as unique as m365 mail item IDs themselves.
ItemRef string `json:"itemRef,omitempty"`
// Indicates the item was added or updated in this backup
// Always `true` for full backups
Updated bool `json:"updated"`

81
src/pkg/backup/details/details_test.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"io"
"strings"
"testing"
"time"
@ -13,6 +14,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path"
@ -48,6 +50,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemRef: "itemref",
},
expectHs: []string{"ID"},
expectVs: []string{"deadbeef"},
@ -58,6 +61,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: ExchangeEvent,
@ -78,6 +82,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: ExchangeContact,
@ -94,6 +99,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: ExchangeMail,
@ -114,6 +120,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{
SharePoint: &SharePointInfo{
ItemName: "itemName",
@ -145,6 +152,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{
ItemName: "itemName",
@ -187,6 +195,7 @@ func exchangeEntry(t *testing.T, id string, size int, it ItemType) DetailsEntry
ShortRef: rr.ShortRef(),
ParentRef: rr.ToBuilder().Dir().ShortRef(),
LocationRef: rr.Folder(true),
ItemRef: rr.Item(),
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: it,
@ -248,11 +257,14 @@ func oneDriveishEntry(t *testing.T, id string, size int, it ItemType) DetailsEnt
ShortRef: rr.ShortRef(),
ParentRef: rr.ToBuilder().Dir().ShortRef(),
LocationRef: loc.String(),
ItemRef: rr.Item(),
ItemInfo: info,
}
}
func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() {
itemID := "foo"
t := suite.T()
table := []struct {
name string
@ -266,23 +278,23 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() {
}{
{
name: "Exchange Email",
entry: exchangeEntry(t, "foo", 42, ExchangeMail),
entry: exchangeEntry(t, itemID, 42, ExchangeMail),
shortRefEqual: assert.Equal,
},
{
name: "OneDrive File",
entry: oneDriveishEntry(t, "foo", 42, OneDriveItem),
entry: oneDriveishEntry(t, itemID, 42, OneDriveItem),
shortRefEqual: assert.NotEqual,
},
{
name: "SharePoint File",
entry: oneDriveishEntry(t, "foo", 42, SharePointLibrary),
entry: oneDriveishEntry(t, itemID, 42, SharePointLibrary),
shortRefEqual: assert.NotEqual,
},
{
name: "Legacy SharePoint File",
entry: func() DetailsEntry {
res := oneDriveishEntry(t, "foo", 42, SharePointLibrary)
res := oneDriveishEntry(t, itemID, 42, SharePointLibrary)
res.SharePoint.ItemType = OneDriveItem
return res
@ -734,6 +746,7 @@ var pathItemsTable = []struct {
{
RepoRef: "abcde",
LocationRef: "locationref",
ItemRef: "itemref",
},
},
expectRepoRefs: []string{"abcde"},
@ -745,10 +758,12 @@ var pathItemsTable = []struct {
{
RepoRef: "abcde",
LocationRef: "locationref",
ItemRef: "itemref",
},
{
RepoRef: "12345",
LocationRef: "locationref2",
ItemRef: "itemref2",
},
},
expectRepoRefs: []string{"abcde", "12345"},
@ -760,10 +775,12 @@ var pathItemsTable = []struct {
{
RepoRef: "abcde",
LocationRef: "locationref",
ItemRef: "itemref",
},
{
RepoRef: "12345",
LocationRef: "locationref2",
ItemRef: "itemref2",
},
{
RepoRef: "deadbeef",
@ -788,6 +805,7 @@ var pathItemsTable = []struct {
{
RepoRef: "foo.meta",
LocationRef: "locationref.dirmeta",
ItemRef: "itemref.meta",
ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: false},
},
@ -795,6 +813,7 @@ var pathItemsTable = []struct {
{
RepoRef: "is-meta-file",
LocationRef: "locationref-meta-file",
ItemRef: "itemref-meta-file",
ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: true},
},
@ -809,14 +828,17 @@ var pathItemsTable = []struct {
{
RepoRef: "abcde",
LocationRef: "locationref",
ItemRef: "itemref",
},
{
RepoRef: "12345",
LocationRef: "locationref2",
ItemRef: "itemref2",
},
{
RepoRef: "foo.meta",
LocationRef: "locationref.dirmeta",
ItemRef: "itemref.dirmeta",
ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: false},
},
@ -824,6 +846,7 @@ var pathItemsTable = []struct {
{
RepoRef: "is-meta-file",
LocationRef: "locationref-meta-file",
ItemRef: "itemref-meta-file",
ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: true},
},
@ -831,6 +854,7 @@ var pathItemsTable = []struct {
{
RepoRef: "deadbeef",
LocationRef: "locationref3",
ItemRef: "itemref3",
ItemInfo: ItemInfo{
Folder: &FolderInfo{
DisplayName: "test folder",
@ -912,7 +936,7 @@ func (suite *DetailsUnitSuite) TestDetailsModel_FilterMetaFiles() {
assert.Len(t, d.Entries, 3)
}
func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() {
t := suite.T()
b := Builder{}
@ -937,8 +961,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
"root:",
"folder",
name + "-id",
},
)
})
otherItemPath := makeItemPath(
t,
@ -952,8 +975,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
"folder",
name + "-id",
name,
},
)
})
err := b.Add(
itemPath,
@ -961,7 +983,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
&path.Builder{},
false,
info)
require.NoError(t, err)
require.NoError(t, err, clues.ToCore(err))
items := b.Details().Items()
require.Len(t, items, 1)
@ -971,6 +993,45 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
assert.NotEqual(t, otherItemPath.ShortRef(), items[0].ShortRef, "same ShortRef as subfolder item")
}
func (suite *DetailsUnitSuite) TestBuilder_Add_cleansFileIDSuffixes() {
var (
t = suite.T()
b = Builder{}
svc = path.OneDriveService
cat = path.FilesCategory
info = ItemInfo{
OneDrive: &OneDriveInfo{
ItemType: OneDriveItem,
ItemName: "in",
DriveName: "dn",
DriveID: "d",
},
}
dataSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.DataFileSuffix})
dirMetaSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.DirMetaFileSuffix})
metaSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.MetaFileSuffix})
)
// Don't need to generate folders for this entry, we just want the itemRef
loc := &path.Builder{}
err := b.Add(dataSfx, loc, false, info)
require.NoError(t, err, clues.ToCore(err))
err = b.Add(dirMetaSfx, loc, false, info)
require.NoError(t, err, clues.ToCore(err))
err = b.Add(metaSfx, loc, false, info)
require.NoError(t, err, clues.ToCore(err))
for _, ent := range b.Details().Items() {
assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.DirMetaFileSuffix))
assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.MetaFileSuffix))
assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.DataFileSuffix))
}
}
func makeItemPath(
t *testing.T,
service path.ServiceType,

13
src/pkg/backup/details/testdata/testdata.go vendored
View File

@ -60,6 +60,7 @@ var (
RepoRef: ExchangeEmailItemPath1.String(),
ShortRef: ExchangeEmailItemPath1.ShortRef(),
ParentRef: ExchangeEmailItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath1.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail,
@ -73,6 +74,7 @@ var (
RepoRef: ExchangeEmailItemPath2.String(),
ShortRef: ExchangeEmailItemPath2.ShortRef(),
ParentRef: ExchangeEmailItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail,
@ -86,6 +88,7 @@ var (
RepoRef: ExchangeEmailItemPath3.String(),
ShortRef: ExchangeEmailItemPath3.ShortRef(),
ParentRef: ExchangeEmailItemPath3.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath3.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail,
@ -108,6 +111,7 @@ var (
RepoRef: ExchangeContactsItemPath1.String(),
ShortRef: ExchangeContactsItemPath1.ShortRef(),
ParentRef: ExchangeContactsItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath1.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeContact,
@ -119,6 +123,7 @@ var (
RepoRef: ExchangeContactsItemPath2.String(),
ShortRef: ExchangeContactsItemPath2.ShortRef(),
ParentRef: ExchangeContactsItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeContact,
@ -139,6 +144,7 @@ var (
RepoRef: ExchangeEventsItemPath1.String(),
ShortRef: ExchangeEventsItemPath1.ShortRef(),
ParentRef: ExchangeEventsItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeEvent,
@ -153,6 +159,7 @@ var (
RepoRef: ExchangeEventsItemPath2.String(),
ShortRef: ExchangeEventsItemPath2.ShortRef(),
ParentRef: ExchangeEventsItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeEvent,
@ -183,6 +190,7 @@ var (
RepoRef: OneDriveItemPath1.String(),
ShortRef: OneDriveItemPath1.ShortRef(),
ParentRef: OneDriveItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: OneDriveItemPath1.Item(),
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -199,6 +207,7 @@ var (
RepoRef: OneDriveItemPath2.String(),
ShortRef: OneDriveItemPath2.ShortRef(),
ParentRef: OneDriveItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: OneDriveItemPath2.Item(),
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -215,6 +224,7 @@ var (
RepoRef: OneDriveItemPath3.String(),
ShortRef: OneDriveItemPath3.ShortRef(),
ParentRef: OneDriveItemPath3.ToBuilder().Dir().ShortRef(),
ItemRef: OneDriveItemPath3.Item(),
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -247,6 +257,7 @@ var (
RepoRef: SharePointLibraryItemPath1.String(),
ShortRef: SharePointLibraryItemPath1.ShortRef(),
ParentRef: SharePointLibraryItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: SharePointLibraryItemPath1.Item(),
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,
@ -263,6 +274,7 @@ var (
RepoRef: SharePointLibraryItemPath2.String(),
ShortRef: SharePointLibraryItemPath2.ShortRef(),
ParentRef: SharePointLibraryItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: SharePointLibraryItemPath2.Item(),
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,
@ -279,6 +291,7 @@ var (
RepoRef: SharePointLibraryItemPath3.String(),
ShortRef: SharePointLibraryItemPath3.ShortRef(),
ParentRef: SharePointLibraryItemPath3.ToBuilder().Dir().ShortRef(),
ItemRef: SharePointLibraryItemPath3.Item(),
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,

1
src/pkg/selectors/example_selectors_test.go
View File

@ -125,6 +125,7 @@ var (
{
RepoRef: "tID/exchange/your-user-id/email/example/itemID",
ShortRef: "xyz",
ItemRef: "123",
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail,

8
src/pkg/selectors/exchange.go
View File

@ -594,6 +594,7 @@ func (ec exchangeCategory) isLeaf() bool {
func (ec exchangeCategory) pathValues(
repo path.Path,
ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) {
var folderCat, itemCat categorizer
@ -611,9 +612,14 @@ func (ec exchangeCategory) pathValues(
return nil, clues.New("bad exchanageCategory").With("category", ec)
}
item := ent.ItemRef
if len(item) == 0 {
item = repo.Item()
}
result := map[categorizer][]string{
folderCat: {repo.Folder(false)},
itemCat: {repo.Item(), ent.ShortRef},
itemCat: {item, ent.ShortRef},
}
if len(ent.LocationRef) > 0 {

17
src/pkg/selectors/exchange_test.go
View File

@ -728,6 +728,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
ent = details.DetailsEntry{
RepoRef: repo.String(),
ShortRef: short,
ItemRef: mail,
LocationRef: loc,
}
)
@ -769,7 +770,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
scopes := setScopesToDefault(test.scope)
var aMatch bool
for _, scope := range scopes {
pvs, err := ExchangeMail.pathValues(repo, ent)
pvs, err := ExchangeMail.pathValues(repo, ent, Config{})
require.NoError(t, err)
if matchesPathValues(scope, ExchangeMail, pvs) {
@ -1312,14 +1313,17 @@ func (suite *ExchangeSelectorSuite) TestScopesByCategory() {
}
func (suite *ExchangeSelectorSuite) TestPasses() {
short := "thisisahashofsomekind"
entry := details.DetailsEntry{ShortRef: short}
const (
mid = "mailID"
cat = ExchangeMail
)
short := "thisisahashofsomekind"
entry := details.DetailsEntry{
ShortRef: short,
ItemRef: mid,
}
var (
es = NewExchangeRestore(Any())
otherMail = setScopesToDefault(es.Mails(Any(), []string{"smarf"}))
@ -1352,7 +1356,7 @@ func (suite *ExchangeSelectorSuite) TestPasses() {
suite.Run(test.name, func() {
t := suite.T()
pvs, err := cat.pathValues(repo, ent)
pvs, err := cat.pathValues(repo, ent, Config{})
require.NoError(t, err)
result := passes(
@ -1493,9 +1497,10 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() {
ent := details.DetailsEntry{
RepoRef: test.path.String(),
ShortRef: "short",
ItemRef: test.path.Item(),
}
pvs, err := test.cat.pathValues(test.path, ent)
pvs, err := test.cat.pathValues(test.path, ent, Config{})
require.NoError(t, err)
assert.Equal(t, test.expect, pvs)
})

1
src/pkg/selectors/helpers_test.go
View File

@ -60,6 +60,7 @@ func (mc mockCategorizer) isLeaf() bool {
func (mc mockCategorizer) pathValues(
repo path.Path,
ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) {
return map[categorizer][]string{
rootCatStub: {"root"},

14
src/pkg/selectors/onedrive.go
View File

@ -3,12 +3,10 @@ package selectors
import (
"context"
"fmt"
"strings"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
@ -394,6 +392,7 @@ func (c oneDriveCategory) isLeaf() bool {
func (c oneDriveCategory) pathValues(
repo path.Path,
ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) {
if ent.OneDrive == nil {
return nil, clues.New("no OneDrive ItemInfo in details")
@ -402,11 +401,18 @@ func (c oneDriveCategory) pathValues(
// Ignore `drives/<driveID>/root:` for folder comparison
rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String()
itemID := strings.TrimSuffix(repo.Item(), metadata.DataFileSuffix)
item := ent.ItemRef
if len(item) == 0 {
item = repo.Item()
}
if cfg.OnlyMatchItemNames {
item = ent.ItemInfo.OneDrive.ItemName
}
result := map[categorizer][]string{
OneDriveFolder: {rFld},
OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef, itemID},
OneDriveItem: {item, ent.ShortRef},
}
if len(ent.LocationRef) > 0 {

129
src/pkg/selectors/onedrive_test.go
View File

@ -173,6 +173,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
Entries: []details.DetailsEntry{
{
RepoRef: file,
ItemRef: "file",
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -182,6 +183,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
},
{
RepoRef: file2,
ItemRef: "file2",
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -191,6 +193,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
},
{
RepoRef: file3,
// item ref intentionally blank to assert fallback case
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -211,36 +214,69 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
deets *details.Details
makeSelector func() *OneDriveRestore
expect []string
cfg Config
}{
{
"all",
deets,
func() *OneDriveRestore {
name: "all",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.AllData())
return odr
},
arr(file, file2, file3),
expect: arr(file, file2, file3),
},
{
"only match file",
deets,
func() *OneDriveRestore {
name: "only match file",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"file2"}))
return odr
},
expect: arr(file2),
},
{
name: "id doesn't match name",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"file2"}))
return odr
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "only match file name",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"fileName2"}))
return odr
},
arr(file2),
expect: arr(file2),
cfg: Config{OnlyMatchItemNames: true},
},
{
"only match folder",
deets,
func() *OneDriveRestore {
name: "name doesn't match id",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"fileName2"}))
return odr
},
expect: []string{},
},
{
name: "only match folder",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore([]string{"uid"})
odr.Include(odr.Folders([]string{"folderA/folderB", "folderA/folderC"}))
return odr
},
arr(file, file2),
expect: arr(file, file2),
},
}
for _, test := range table {
@ -251,6 +287,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
defer flush()
sel := test.makeSelector()
sel.Configure(test.cfg)
results := sel.Reduce(ctx, test.deets, fault.New(true))
paths := results.Paths()
assert.Equal(t, test.expect, paths)
@ -262,30 +299,68 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
t := suite.T()
fileName := "file"
fileID := fileName + "-id"
shortRef := "short"
elems := []string{"drive", "driveID", "root:", "dir1", "dir2", fileName + "-id"}
elems := []string{"drive", "driveID", "root:", "dir1", "dir2", fileID}
filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...)
require.NoError(t, err, clues.ToCore(err))
expected := map[categorizer][]string{
OneDriveFolder: {"dir1/dir2"},
OneDriveItem: {fileName, shortRef, fileName + "-id"},
}
ent := details.DetailsEntry{
RepoRef: filePath.String(),
ShortRef: shortRef,
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemName: fileName,
table := []struct {
name string
pathElems []string
expected map[categorizer][]string
cfg Config
}{
{
name: "items",
pathElems: elems,
expected: map[categorizer][]string{
OneDriveFolder: {"dir1/dir2"},
OneDriveItem: {fileID, shortRef},
},
cfg: Config{},
},
{
name: "items w/ name",
pathElems: elems,
expected: map[categorizer][]string{
OneDriveFolder: {"dir1/dir2"},
OneDriveItem: {fileName, shortRef},
},
cfg: Config{OnlyMatchItemNames: true},
},
}
r, err := OneDriveItem.pathValues(filePath, ent)
require.NoError(t, err)
assert.Equal(t, expected, r)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
itemPath, err := path.Build(
"tenant",
"site",
path.OneDriveService,
path.FilesCategory,
true,
test.pathElems...)
require.NoError(t, err, clues.ToCore(err))
ent := details.DetailsEntry{
RepoRef: filePath.String(),
ShortRef: shortRef,
ItemRef: fileID,
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemName: fileName,
},
},
}
pv, err := OneDriveItem.pathValues(itemPath, ent, test.cfg)
require.NoError(t, err)
assert.Equal(t, test.expected, pv)
})
}
}
func (suite *OneDriveSelectorSuite) TestOneDriveScope_MatchesInfo() {

4
src/pkg/selectors/scopes.go
View File

@ -89,7 +89,7 @@ type (
// folderCat: folder,
// itemCat: itemID,
// }
pathValues(path.Path, details.DetailsEntry) (map[categorizer][]string, error)
pathValues(path.Path, details.DetailsEntry, Config) (map[categorizer][]string, error)
// pathKeys produces a list of categorizers that can be used as keys in the pathValues
// map. The combination of the two funcs generically interprets the context of the
@ -389,7 +389,7 @@ func reduce[T scopeT, C categoryT](
continue
}
pv, err := dc.pathValues(repoPath, *ent)
pv, err := dc.pathValues(repoPath, *ent, s.Cfg)
if err != nil {
el.AddRecoverable(clues.Wrap(err, "getting path values").WithClues(ictx))
continue

2
src/pkg/selectors/scopes_test.go
View File

@ -366,7 +366,7 @@ func (suite *SelectorScopesSuite) TestPasses() {
}
)
pvs, err := cat.pathValues(pth, entry)
pvs, err := cat.pathValues(pth, entry, Config{})
require.NoError(suite.T(), err)
for _, test := range reduceTestTable {

15
src/pkg/selectors/selectors.go
View File

@ -122,6 +122,16 @@ type Selector struct {
// A slice of inclusion scopes. Comparators must match either one of these,
// or all filters, to be included.
Includes []scope `json:"includes,omitempty"`
Cfg Config `json:"cfg,omitempty"`
}
// Config defines broad-scale selector behavior.
type Config struct {
// OnlyMatchItemNames tells the reducer to ignore matching on itemRef values
// and other item IDs in favor of matching the item name. Normal behavior only
// matches on itemRefs.
OnlyMatchItemNames bool
}
// helper for specific selector instance constructors.
@ -140,6 +150,11 @@ func newSelector(s service, resourceOwners []string) Selector {
}
}
// Configure sets the selector configuration.
func (s *Selector) Configure(cfg Config) {
s.Cfg = cfg
}
// DiscreteResourceOwners returns the list of individual resourceOwners used
// in the selector.
// TODO(rkeepers): remove in favor of split and s.DiscreteOwner

17
src/pkg/selectors/sharepoint.go
View File

@ -3,12 +3,10 @@ package selectors
import (
"context"
"fmt"
"strings"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
@ -519,10 +517,10 @@ func (c sharePointCategory) isLeaf() bool {
func (c sharePointCategory) pathValues(
repo path.Path,
ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) {
var (
folderCat, itemCat categorizer
itemName = repo.Item()
dropDriveFolderPrefix bool
itemID string
)
@ -535,8 +533,6 @@ func (c sharePointCategory) pathValues(
dropDriveFolderPrefix = true
folderCat, itemCat = SharePointLibraryFolder, SharePointLibraryItem
itemID = strings.TrimSuffix(itemName, metadata.DataFileSuffix)
itemName = ent.SharePoint.ItemName
case SharePointList, SharePointListItem:
folderCat, itemCat = SharePointList, SharePointListItem
@ -554,9 +550,18 @@ func (c sharePointCategory) pathValues(
rFld = path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String()
}
item := ent.ItemRef
if len(item) == 0 {
item = repo.Item()
}
if cfg.OnlyMatchItemNames {
item = ent.ItemInfo.SharePoint.ItemName
}
result := map[categorizer][]string{
folderCat: {rFld},
itemCat: {itemName, ent.ShortRef},
itemCat: {item, ent.ShortRef},
}
if len(itemID) > 0 {

64
src/pkg/selectors/sharepoint_test.go
View File

@ -220,6 +220,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
Entries: []details.DetailsEntry{
{
RepoRef: item,
ItemRef: "item",
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,
@ -229,6 +230,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
},
{
RepoRef: item2,
// ItemRef intentionally blank to test fallback case
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,
@ -238,6 +240,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
},
{
RepoRef: item3,
ItemRef: "item3",
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,
@ -247,6 +250,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
},
{
RepoRef: item4,
ItemRef: "item4",
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointPage,
@ -256,6 +260,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
},
{
RepoRef: item5,
// ItemRef intentionally blank to test fallback case
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointPage,
@ -276,6 +281,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
deets *details.Details
makeSelector func() *SharePointRestore
expect []string
cfg Config
}{
{
name: "all",
@ -290,12 +296,44 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
{
name: "only match item",
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"item2"}))
return odr
},
expect: arr(item2),
},
{
name: "id doesn't match name",
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"item2"}))
return odr
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "only match item name",
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"itemName2"}))
return odr
},
expect: arr(item2),
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "name doesn't match",
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"itemName2"}))
return odr
},
expect: []string{},
},
{
name: "only match folder",
@ -326,6 +364,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
defer flush()
sel := test.makeSelector()
sel.Configure(test.cfg)
results := sel.Reduce(ctx, test.deets, fault.New(true))
paths := results.Paths()
assert.Equal(t, test.expect, paths)
@ -336,9 +375,10 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
var (
itemName = "item"
itemID = "item-id"
shortRef = "short"
driveElems = []string{"drive", "drive!id", "root:", "dir1", "dir2", itemName + "-id"}
elems = []string{"dir1", "dir2", itemName + "-id"}
driveElems = []string{"drive", "drive!id", "root:", "dir1", "dir2", itemID}
elems = []string{"dir1", "dir2", itemID}
)
table := []struct {
@ -346,6 +386,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
sc sharePointCategory
pathElems []string
expected map[categorizer][]string
cfg Config
}{
{
name: "SharePoint Libraries",
@ -353,8 +394,19 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
pathElems: driveElems,
expected: map[categorizer][]string{
SharePointLibraryFolder: {"dir1/dir2"},
SharePointLibraryItem: {itemName, shortRef, itemName + "-id"},
SharePointLibraryItem: {itemID, shortRef},
},
cfg: Config{},
},
{
name: "SharePoint Libraries w/ name",
sc: SharePointLibraryItem,
pathElems: driveElems,
expected: map[categorizer][]string{
SharePointLibraryFolder: {"dir1/dir2"},
SharePointLibraryItem: {itemName, shortRef},
},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "SharePoint Lists",
@ -362,8 +414,9 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
pathElems: elems,
expected: map[categorizer][]string{
SharePointList: {"dir1/dir2"},
SharePointListItem: {"item-id", shortRef},
SharePointListItem: {itemID, shortRef},
},
cfg: Config{},
},
}
@ -383,6 +436,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
ent := details.DetailsEntry{
RepoRef: itemPath.String(),
ShortRef: shortRef,
ItemRef: itemPath.Item(),
ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemName: itemName,
@ -390,7 +444,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
},
}
pv, err := test.sc.pathValues(itemPath, ent)
pv, err := test.sc.pathValues(itemPath, ent, test.cfg)
require.NoError(t, err)
assert.Equal(t, test.expected, pv)
})