add itemref to details entry (#3160)

Adds an new details entry field: itemRef.
This holds a stable, semi-unique identifier to the item represented by that entry.

---

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3027

#### Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-04-19 17:25:11 -06:00 committed by GitHub
parent 2121b40293
commit 306db14339
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 359 additions and 78 deletions

View File

@ -190,6 +190,9 @@ func handleDeleteCmd(cmd *cobra.Command, args []string) error {
// common handlers // common handlers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// standard set of selector behavior that we want used in the cli
var defaultSelectorConfig = selectors.Config{OnlyMatchItemNames: true}
func runBackups( func runBackups(
ctx context.Context, ctx context.Context,
r repository.Repository, r repository.Repository,
@ -203,6 +206,8 @@ func runBackups(
) )
for _, discSel := range selectorSet { for _, discSel := range selectorSet {
discSel.Configure(defaultSelectorConfig)
var ( var (
owner = discSel.DiscreteOwner owner = discSel.DiscreteOwner
ictx = clues.Add(ctx, "resource_owner", owner) ictx = clues.Add(ctx, "resource_owner", owner)

View File

@ -317,6 +317,7 @@ func runDetailsExchangeCmd(
if !skipReduce { if !skipReduce {
sel := utils.IncludeExchangeRestoreDataSelectors(opts) sel := utils.IncludeExchangeRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterExchangeRestoreInfoSelectors(sel, opts) utils.FilterExchangeRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs) d = sel.Reduce(ctx, d, errs)
} }

View File

@ -278,6 +278,7 @@ func runDetailsOneDriveCmd(
if !skipReduce { if !skipReduce {
sel := utils.IncludeOneDriveRestoreDataSelectors(opts) sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterOneDriveRestoreInfoSelectors(sel, opts) utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs) d = sel.Reduce(ctx, d, errs)
} }

View File

@ -362,6 +362,7 @@ func runDetailsSharePointCmd(
if !skipReduce { if !skipReduce {
sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterSharePointRestoreInfoSelectors(sel, opts) utils.FilterSharePointRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs) d = sel.Reduce(ctx, d, errs)
} }

View File

@ -2,14 +2,12 @@ package testdata
import ( import (
"context" "context"
"strings"
"time" "time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/backup/details/testdata"
@ -201,10 +199,10 @@ var (
}, },
}, },
{ {
Name: "MailID", Name: "MailItemRef",
Expected: []details.DetailsEntry{testdata.ExchangeEmailItems[0]}, Expected: []details.DetailsEntry{testdata.ExchangeEmailItems[0]},
Opts: utils.ExchangeOpts{ Opts: utils.ExchangeOpts{
Email: []string{testdata.ExchangeEmailItemPath1.Item()}, Email: []string{testdata.ExchangeEmailItems[0].ItemRef},
}, },
}, },
{ {
@ -413,13 +411,11 @@ var (
}, },
}, },
{ {
Name: "SelectRepoItemName", Name: "ItemRefMatchesNothing",
Expected: []details.DetailsEntry{ Expected: []details.DetailsEntry{},
testdata.OneDriveItems[0],
},
Opts: utils.OneDriveOpts{ Opts: utils.OneDriveOpts{
FileName: []string{ FileName: []string{
strings.TrimSuffix(testdata.OneDriveItemPath1.Item(), metadata.DataFileSuffix), testdata.OneDriveItems[0].ItemRef,
}, },
}, },
}, },
@ -534,13 +530,11 @@ var (
}, },
}, },
{ {
Name: "SelectRepoItemName", Name: "ItemRefMatchesNothing",
Expected: []details.DetailsEntry{ Expected: []details.DetailsEntry{},
testdata.SharePointLibraryItems[0],
},
Opts: utils.SharePointOpts{ Opts: utils.SharePointOpts{
FileName: []string{ FileName: []string{
strings.TrimSuffix(testdata.SharePointLibraryItemPath1.Item(), metadata.DataFileSuffix), testdata.SharePointLibraryItems[0].ItemRef,
}, },
}, },
}, },

View File

@ -903,8 +903,7 @@ func traverseBaseDir(
oldDirPath, oldDirPath,
currentPath, currentPath,
dEntry, dEntry,
roots, roots)
)
}) })
if err != nil { if err != nil {
return clues.Wrap(err, "traversing base directory") return clues.Wrap(err, "traversing base directory")

View File

@ -481,11 +481,17 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
require.Len(t, cp.pending, len(ci)) require.Len(t, cp.pending, len(ci))
foundItems := map[string]bool{}
for k, v := range ci { for k, v := range ci {
if cachedTest.cached { if cachedTest.cached {
cp.CachedFile(k, v.totalBytes) cp.CachedFile(k, v.totalBytes)
} }
if v.info != nil && v.info.repoPath != nil {
foundItems[v.info.repoPath.Item()] = false
}
cp.FinishedFile(k, v.err) cp.FinishedFile(k, v.err)
} }
@ -496,6 +502,14 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
for _, entry := range entries { for _, entry := range entries {
assert.Equal(t, !cachedTest.cached, entry.Updated) assert.Equal(t, !cachedTest.cached, entry.Updated)
foundItems[entry.ItemRef] = true
}
if test.expectedNumEntries > 0 {
for item, found := range foundItems {
assert.Truef(t, found, "details missing item: %s", item)
}
} }
}) })
} }

View File

@ -320,6 +320,7 @@ func makeDetailsEntry(
RepoRef: p.String(), RepoRef: p.String(),
ShortRef: p.ShortRef(), ShortRef: p.ShortRef(),
ParentRef: p.ToBuilder().Dir().ShortRef(), ParentRef: p.ToBuilder().Dir().ShortRef(),
ItemRef: p.Item(),
LocationRef: lr, LocationRef: lr,
ItemInfo: details.ItemInfo{}, ItemInfo: details.ItemInfo{},
Updated: updated, Updated: updated,

View File

@ -191,6 +191,7 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef) assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef)
assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef) assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef)
assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef) assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef)
assert.Equal(t, deets.Entries[0].ItemRef, readDeets.Entries[0].ItemRef)
assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated) assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated)
assert.NotNil(t, readDeets.Entries[0].Exchange) assert.NotNil(t, readDeets.Entries[0].Exchange)
assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange) assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange)

View File

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"io" "io"
"strconv" "strconv"
"strings"
"sync" "sync"
"time" "time"
@ -14,6 +15,7 @@ import (
"github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
@ -388,6 +390,7 @@ func (d *Details) add(
ShortRef: repoRef.ShortRef(), ShortRef: repoRef.ShortRef(),
ParentRef: repoRef.ToBuilder().Dir().ShortRef(), ParentRef: repoRef.ToBuilder().Dir().ShortRef(),
LocationRef: locationRef.String(), LocationRef: locationRef.String(),
ItemRef: repoRef.Item(),
Updated: updated, Updated: updated,
ItemInfo: info, ItemInfo: info,
} }
@ -418,6 +421,9 @@ func (d *Details) add(
elements := repoRef.Elements() elements := repoRef.Elements()
elements = append(elements[:len(elements)-1], filename, repoRef.Item()) elements = append(elements[:len(elements)-1], filename, repoRef.Item())
entry.ShortRef = path.Builder{}.Append(elements...).ShortRef() entry.ShortRef = path.Builder{}.Append(elements...).ShortRef()
// clean metadata suffixes from item refs
entry.ItemRef = withoutMetadataSuffix(entry.ItemRef)
} }
d.Entries = append(d.Entries, entry) d.Entries = append(d.Entries, entry)
@ -437,6 +443,16 @@ func UnmarshalTo(d *Details) func(io.ReadCloser) error {
} }
} }
// remove metadata file suffixes from the string.
// assumes only one suffix is applied to any given id.
func withoutMetadataSuffix(id string) string {
id = strings.TrimSuffix(id, metadata.DirMetaFileSuffix)
id = strings.TrimSuffix(id, metadata.MetaFileSuffix)
id = strings.TrimSuffix(id, metadata.DataFileSuffix)
return id
}
// -------------------------------------------------------------------------------- // --------------------------------------------------------------------------------
// Entry // Entry
// -------------------------------------------------------------------------------- // --------------------------------------------------------------------------------
@ -458,6 +474,12 @@ type DetailsEntry struct {
// Currently only implemented for Exchange Calendars. // Currently only implemented for Exchange Calendars.
LocationRef string `json:"locationRef,omitempty"` LocationRef string `json:"locationRef,omitempty"`
// ItemRef contains the stable id of the item itself. ItemRef is not
// guaranteed to be unique within a repository. Uniqueness guarantees
// maximally inherit from the source item. Eg: Entries for m365 mail items
// are only as unique as m365 mail item IDs themselves.
ItemRef string `json:"itemRef,omitempty"`
// Indicates the item was added or updated in this backup // Indicates the item was added or updated in this backup
// Always `true` for full backups // Always `true` for full backups
Updated bool `json:"updated"` Updated bool `json:"updated"`

View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"io" "io"
"strings"
"testing" "testing"
"time" "time"
@ -13,6 +14,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -48,6 +50,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef", RepoRef: "reporef",
ShortRef: "deadbeef", ShortRef: "deadbeef",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
}, },
expectHs: []string{"ID"}, expectHs: []string{"ID"},
expectVs: []string{"deadbeef"}, expectVs: []string{"deadbeef"},
@ -58,6 +61,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef", RepoRef: "reporef",
ShortRef: "deadbeef", ShortRef: "deadbeef",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{ Exchange: &ExchangeInfo{
ItemType: ExchangeEvent, ItemType: ExchangeEvent,
@ -78,6 +82,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef", RepoRef: "reporef",
ShortRef: "deadbeef", ShortRef: "deadbeef",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{ Exchange: &ExchangeInfo{
ItemType: ExchangeContact, ItemType: ExchangeContact,
@ -94,6 +99,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef", RepoRef: "reporef",
ShortRef: "deadbeef", ShortRef: "deadbeef",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{ Exchange: &ExchangeInfo{
ItemType: ExchangeMail, ItemType: ExchangeMail,
@ -114,6 +120,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef", RepoRef: "reporef",
ShortRef: "deadbeef", ShortRef: "deadbeef",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
SharePoint: &SharePointInfo{ SharePoint: &SharePointInfo{
ItemName: "itemName", ItemName: "itemName",
@ -145,6 +152,7 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
RepoRef: "reporef", RepoRef: "reporef",
ShortRef: "deadbeef", ShortRef: "deadbeef",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{ OneDrive: &OneDriveInfo{
ItemName: "itemName", ItemName: "itemName",
@ -187,6 +195,7 @@ func exchangeEntry(t *testing.T, id string, size int, it ItemType) DetailsEntry
ShortRef: rr.ShortRef(), ShortRef: rr.ShortRef(),
ParentRef: rr.ToBuilder().Dir().ShortRef(), ParentRef: rr.ToBuilder().Dir().ShortRef(),
LocationRef: rr.Folder(true), LocationRef: rr.Folder(true),
ItemRef: rr.Item(),
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{ Exchange: &ExchangeInfo{
ItemType: it, ItemType: it,
@ -248,11 +257,14 @@ func oneDriveishEntry(t *testing.T, id string, size int, it ItemType) DetailsEnt
ShortRef: rr.ShortRef(), ShortRef: rr.ShortRef(),
ParentRef: rr.ToBuilder().Dir().ShortRef(), ParentRef: rr.ToBuilder().Dir().ShortRef(),
LocationRef: loc.String(), LocationRef: loc.String(),
ItemRef: rr.Item(),
ItemInfo: info, ItemInfo: info,
} }
} }
func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() { func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() {
itemID := "foo"
t := suite.T() t := suite.T()
table := []struct { table := []struct {
name string name string
@ -266,23 +278,23 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_NoLocationFolders() {
}{ }{
{ {
name: "Exchange Email", name: "Exchange Email",
entry: exchangeEntry(t, "foo", 42, ExchangeMail), entry: exchangeEntry(t, itemID, 42, ExchangeMail),
shortRefEqual: assert.Equal, shortRefEqual: assert.Equal,
}, },
{ {
name: "OneDrive File", name: "OneDrive File",
entry: oneDriveishEntry(t, "foo", 42, OneDriveItem), entry: oneDriveishEntry(t, itemID, 42, OneDriveItem),
shortRefEqual: assert.NotEqual, shortRefEqual: assert.NotEqual,
}, },
{ {
name: "SharePoint File", name: "SharePoint File",
entry: oneDriveishEntry(t, "foo", 42, SharePointLibrary), entry: oneDriveishEntry(t, itemID, 42, SharePointLibrary),
shortRefEqual: assert.NotEqual, shortRefEqual: assert.NotEqual,
}, },
{ {
name: "Legacy SharePoint File", name: "Legacy SharePoint File",
entry: func() DetailsEntry { entry: func() DetailsEntry {
res := oneDriveishEntry(t, "foo", 42, SharePointLibrary) res := oneDriveishEntry(t, itemID, 42, SharePointLibrary)
res.SharePoint.ItemType = OneDriveItem res.SharePoint.ItemType = OneDriveItem
return res return res
@ -734,6 +746,7 @@ var pathItemsTable = []struct {
{ {
RepoRef: "abcde", RepoRef: "abcde",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
}, },
}, },
expectRepoRefs: []string{"abcde"}, expectRepoRefs: []string{"abcde"},
@ -745,10 +758,12 @@ var pathItemsTable = []struct {
{ {
RepoRef: "abcde", RepoRef: "abcde",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
}, },
{ {
RepoRef: "12345", RepoRef: "12345",
LocationRef: "locationref2", LocationRef: "locationref2",
ItemRef: "itemref2",
}, },
}, },
expectRepoRefs: []string{"abcde", "12345"}, expectRepoRefs: []string{"abcde", "12345"},
@ -760,10 +775,12 @@ var pathItemsTable = []struct {
{ {
RepoRef: "abcde", RepoRef: "abcde",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
}, },
{ {
RepoRef: "12345", RepoRef: "12345",
LocationRef: "locationref2", LocationRef: "locationref2",
ItemRef: "itemref2",
}, },
{ {
RepoRef: "deadbeef", RepoRef: "deadbeef",
@ -788,6 +805,7 @@ var pathItemsTable = []struct {
{ {
RepoRef: "foo.meta", RepoRef: "foo.meta",
LocationRef: "locationref.dirmeta", LocationRef: "locationref.dirmeta",
ItemRef: "itemref.meta",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: false}, OneDrive: &OneDriveInfo{IsMeta: false},
}, },
@ -795,6 +813,7 @@ var pathItemsTable = []struct {
{ {
RepoRef: "is-meta-file", RepoRef: "is-meta-file",
LocationRef: "locationref-meta-file", LocationRef: "locationref-meta-file",
ItemRef: "itemref-meta-file",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: true}, OneDrive: &OneDriveInfo{IsMeta: true},
}, },
@ -809,14 +828,17 @@ var pathItemsTable = []struct {
{ {
RepoRef: "abcde", RepoRef: "abcde",
LocationRef: "locationref", LocationRef: "locationref",
ItemRef: "itemref",
}, },
{ {
RepoRef: "12345", RepoRef: "12345",
LocationRef: "locationref2", LocationRef: "locationref2",
ItemRef: "itemref2",
}, },
{ {
RepoRef: "foo.meta", RepoRef: "foo.meta",
LocationRef: "locationref.dirmeta", LocationRef: "locationref.dirmeta",
ItemRef: "itemref.dirmeta",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: false}, OneDrive: &OneDriveInfo{IsMeta: false},
}, },
@ -824,6 +846,7 @@ var pathItemsTable = []struct {
{ {
RepoRef: "is-meta-file", RepoRef: "is-meta-file",
LocationRef: "locationref-meta-file", LocationRef: "locationref-meta-file",
ItemRef: "itemref-meta-file",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{IsMeta: true}, OneDrive: &OneDriveInfo{IsMeta: true},
}, },
@ -831,6 +854,7 @@ var pathItemsTable = []struct {
{ {
RepoRef: "deadbeef", RepoRef: "deadbeef",
LocationRef: "locationref3", LocationRef: "locationref3",
ItemRef: "itemref3",
ItemInfo: ItemInfo{ ItemInfo: ItemInfo{
Folder: &FolderInfo{ Folder: &FolderInfo{
DisplayName: "test folder", DisplayName: "test folder",
@ -912,7 +936,7 @@ func (suite *DetailsUnitSuite) TestDetailsModel_FilterMetaFiles() {
assert.Len(t, d.Entries, 3) assert.Len(t, d.Entries, 3)
} }
func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() { func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() {
t := suite.T() t := suite.T()
b := Builder{} b := Builder{}
@ -937,8 +961,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
"root:", "root:",
"folder", "folder",
name + "-id", name + "-id",
}, })
)
otherItemPath := makeItemPath( otherItemPath := makeItemPath(
t, t,
@ -952,8 +975,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
"folder", "folder",
name + "-id", name + "-id",
name, name,
}, })
)
err := b.Add( err := b.Add(
itemPath, itemPath,
@ -961,7 +983,7 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
&path.Builder{}, &path.Builder{},
false, false,
info) info)
require.NoError(t, err) require.NoError(t, err, clues.ToCore(err))
items := b.Details().Items() items := b.Details().Items()
require.Len(t, items, 1) require.Len(t, items, 1)
@ -971,6 +993,45 @@ func (suite *DetailsUnitSuite) TestDetails_Add_ShortRefs_Unique_From_Folder() {
assert.NotEqual(t, otherItemPath.ShortRef(), items[0].ShortRef, "same ShortRef as subfolder item") assert.NotEqual(t, otherItemPath.ShortRef(), items[0].ShortRef, "same ShortRef as subfolder item")
} }
func (suite *DetailsUnitSuite) TestBuilder_Add_cleansFileIDSuffixes() {
var (
t = suite.T()
b = Builder{}
svc = path.OneDriveService
cat = path.FilesCategory
info = ItemInfo{
OneDrive: &OneDriveInfo{
ItemType: OneDriveItem,
ItemName: "in",
DriveName: "dn",
DriveID: "d",
},
}
dataSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.DataFileSuffix})
dirMetaSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.DirMetaFileSuffix})
metaSfx = makeItemPath(t, svc, cat, "t", "u", []string{"d", "r:", "f", "i1" + metadata.MetaFileSuffix})
)
// Don't need to generate folders for this entry, we just want the itemRef
loc := &path.Builder{}
err := b.Add(dataSfx, loc, false, info)
require.NoError(t, err, clues.ToCore(err))
err = b.Add(dirMetaSfx, loc, false, info)
require.NoError(t, err, clues.ToCore(err))
err = b.Add(metaSfx, loc, false, info)
require.NoError(t, err, clues.ToCore(err))
for _, ent := range b.Details().Items() {
assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.DirMetaFileSuffix))
assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.MetaFileSuffix))
assert.False(t, strings.HasSuffix(ent.ItemRef, metadata.DataFileSuffix))
}
}
func makeItemPath( func makeItemPath(
t *testing.T, t *testing.T,
service path.ServiceType, service path.ServiceType,

View File

@ -60,6 +60,7 @@ var (
RepoRef: ExchangeEmailItemPath1.String(), RepoRef: ExchangeEmailItemPath1.String(),
ShortRef: ExchangeEmailItemPath1.ShortRef(), ShortRef: ExchangeEmailItemPath1.ShortRef(),
ParentRef: ExchangeEmailItemPath1.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeEmailItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath1.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail, ItemType: details.ExchangeMail,
@ -73,6 +74,7 @@ var (
RepoRef: ExchangeEmailItemPath2.String(), RepoRef: ExchangeEmailItemPath2.String(),
ShortRef: ExchangeEmailItemPath2.ShortRef(), ShortRef: ExchangeEmailItemPath2.ShortRef(),
ParentRef: ExchangeEmailItemPath2.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeEmailItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail, ItemType: details.ExchangeMail,
@ -86,6 +88,7 @@ var (
RepoRef: ExchangeEmailItemPath3.String(), RepoRef: ExchangeEmailItemPath3.String(),
ShortRef: ExchangeEmailItemPath3.ShortRef(), ShortRef: ExchangeEmailItemPath3.ShortRef(),
ParentRef: ExchangeEmailItemPath3.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeEmailItemPath3.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath3.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail, ItemType: details.ExchangeMail,
@ -108,6 +111,7 @@ var (
RepoRef: ExchangeContactsItemPath1.String(), RepoRef: ExchangeContactsItemPath1.String(),
ShortRef: ExchangeContactsItemPath1.ShortRef(), ShortRef: ExchangeContactsItemPath1.ShortRef(),
ParentRef: ExchangeContactsItemPath1.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeContactsItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath1.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeContact, ItemType: details.ExchangeContact,
@ -119,6 +123,7 @@ var (
RepoRef: ExchangeContactsItemPath2.String(), RepoRef: ExchangeContactsItemPath2.String(),
ShortRef: ExchangeContactsItemPath2.ShortRef(), ShortRef: ExchangeContactsItemPath2.ShortRef(),
ParentRef: ExchangeContactsItemPath2.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeContactsItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeContact, ItemType: details.ExchangeContact,
@ -139,6 +144,7 @@ var (
RepoRef: ExchangeEventsItemPath1.String(), RepoRef: ExchangeEventsItemPath1.String(),
ShortRef: ExchangeEventsItemPath1.ShortRef(), ShortRef: ExchangeEventsItemPath1.ShortRef(),
ParentRef: ExchangeEventsItemPath1.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeEventsItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeEvent, ItemType: details.ExchangeEvent,
@ -153,6 +159,7 @@ var (
RepoRef: ExchangeEventsItemPath2.String(), RepoRef: ExchangeEventsItemPath2.String(),
ShortRef: ExchangeEventsItemPath2.ShortRef(), ShortRef: ExchangeEventsItemPath2.ShortRef(),
ParentRef: ExchangeEventsItemPath2.ToBuilder().Dir().ShortRef(), ParentRef: ExchangeEventsItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: ExchangeEmailItemPath2.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeEvent, ItemType: details.ExchangeEvent,
@ -183,6 +190,7 @@ var (
RepoRef: OneDriveItemPath1.String(), RepoRef: OneDriveItemPath1.String(),
ShortRef: OneDriveItemPath1.ShortRef(), ShortRef: OneDriveItemPath1.ShortRef(),
ParentRef: OneDriveItemPath1.ToBuilder().Dir().ShortRef(), ParentRef: OneDriveItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: OneDriveItemPath1.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -199,6 +207,7 @@ var (
RepoRef: OneDriveItemPath2.String(), RepoRef: OneDriveItemPath2.String(),
ShortRef: OneDriveItemPath2.ShortRef(), ShortRef: OneDriveItemPath2.ShortRef(),
ParentRef: OneDriveItemPath2.ToBuilder().Dir().ShortRef(), ParentRef: OneDriveItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: OneDriveItemPath2.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -215,6 +224,7 @@ var (
RepoRef: OneDriveItemPath3.String(), RepoRef: OneDriveItemPath3.String(),
ShortRef: OneDriveItemPath3.ShortRef(), ShortRef: OneDriveItemPath3.ShortRef(),
ParentRef: OneDriveItemPath3.ToBuilder().Dir().ShortRef(), ParentRef: OneDriveItemPath3.ToBuilder().Dir().ShortRef(),
ItemRef: OneDriveItemPath3.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -247,6 +257,7 @@ var (
RepoRef: SharePointLibraryItemPath1.String(), RepoRef: SharePointLibraryItemPath1.String(),
ShortRef: SharePointLibraryItemPath1.ShortRef(), ShortRef: SharePointLibraryItemPath1.ShortRef(),
ParentRef: SharePointLibraryItemPath1.ToBuilder().Dir().ShortRef(), ParentRef: SharePointLibraryItemPath1.ToBuilder().Dir().ShortRef(),
ItemRef: SharePointLibraryItemPath1.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary, ItemType: details.SharePointLibrary,
@ -263,6 +274,7 @@ var (
RepoRef: SharePointLibraryItemPath2.String(), RepoRef: SharePointLibraryItemPath2.String(),
ShortRef: SharePointLibraryItemPath2.ShortRef(), ShortRef: SharePointLibraryItemPath2.ShortRef(),
ParentRef: SharePointLibraryItemPath2.ToBuilder().Dir().ShortRef(), ParentRef: SharePointLibraryItemPath2.ToBuilder().Dir().ShortRef(),
ItemRef: SharePointLibraryItemPath2.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary, ItemType: details.SharePointLibrary,
@ -279,6 +291,7 @@ var (
RepoRef: SharePointLibraryItemPath3.String(), RepoRef: SharePointLibraryItemPath3.String(),
ShortRef: SharePointLibraryItemPath3.ShortRef(), ShortRef: SharePointLibraryItemPath3.ShortRef(),
ParentRef: SharePointLibraryItemPath3.ToBuilder().Dir().ShortRef(), ParentRef: SharePointLibraryItemPath3.ToBuilder().Dir().ShortRef(),
ItemRef: SharePointLibraryItemPath3.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary, ItemType: details.SharePointLibrary,

View File

@ -125,6 +125,7 @@ var (
{ {
RepoRef: "tID/exchange/your-user-id/email/example/itemID", RepoRef: "tID/exchange/your-user-id/email/example/itemID",
ShortRef: "xyz", ShortRef: "xyz",
ItemRef: "123",
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
ItemType: details.ExchangeMail, ItemType: details.ExchangeMail,

View File

@ -594,6 +594,7 @@ func (ec exchangeCategory) isLeaf() bool {
func (ec exchangeCategory) pathValues( func (ec exchangeCategory) pathValues(
repo path.Path, repo path.Path,
ent details.DetailsEntry, ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) { ) (map[categorizer][]string, error) {
var folderCat, itemCat categorizer var folderCat, itemCat categorizer
@ -611,9 +612,14 @@ func (ec exchangeCategory) pathValues(
return nil, clues.New("bad exchanageCategory").With("category", ec) return nil, clues.New("bad exchanageCategory").With("category", ec)
} }
item := ent.ItemRef
if len(item) == 0 {
item = repo.Item()
}
result := map[categorizer][]string{ result := map[categorizer][]string{
folderCat: {repo.Folder(false)}, folderCat: {repo.Folder(false)},
itemCat: {repo.Item(), ent.ShortRef}, itemCat: {item, ent.ShortRef},
} }
if len(ent.LocationRef) > 0 { if len(ent.LocationRef) > 0 {

View File

@ -728,6 +728,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
ent = details.DetailsEntry{ ent = details.DetailsEntry{
RepoRef: repo.String(), RepoRef: repo.String(),
ShortRef: short, ShortRef: short,
ItemRef: mail,
LocationRef: loc, LocationRef: loc,
} }
) )
@ -769,7 +770,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
scopes := setScopesToDefault(test.scope) scopes := setScopesToDefault(test.scope)
var aMatch bool var aMatch bool
for _, scope := range scopes { for _, scope := range scopes {
pvs, err := ExchangeMail.pathValues(repo, ent) pvs, err := ExchangeMail.pathValues(repo, ent, Config{})
require.NoError(t, err) require.NoError(t, err)
if matchesPathValues(scope, ExchangeMail, pvs) { if matchesPathValues(scope, ExchangeMail, pvs) {
@ -1312,14 +1313,17 @@ func (suite *ExchangeSelectorSuite) TestScopesByCategory() {
} }
func (suite *ExchangeSelectorSuite) TestPasses() { func (suite *ExchangeSelectorSuite) TestPasses() {
short := "thisisahashofsomekind"
entry := details.DetailsEntry{ShortRef: short}
const ( const (
mid = "mailID" mid = "mailID"
cat = ExchangeMail cat = ExchangeMail
) )
short := "thisisahashofsomekind"
entry := details.DetailsEntry{
ShortRef: short,
ItemRef: mid,
}
var ( var (
es = NewExchangeRestore(Any()) es = NewExchangeRestore(Any())
otherMail = setScopesToDefault(es.Mails(Any(), []string{"smarf"})) otherMail = setScopesToDefault(es.Mails(Any(), []string{"smarf"}))
@ -1352,7 +1356,7 @@ func (suite *ExchangeSelectorSuite) TestPasses() {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() t := suite.T()
pvs, err := cat.pathValues(repo, ent) pvs, err := cat.pathValues(repo, ent, Config{})
require.NoError(t, err) require.NoError(t, err)
result := passes( result := passes(
@ -1493,9 +1497,10 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() {
ent := details.DetailsEntry{ ent := details.DetailsEntry{
RepoRef: test.path.String(), RepoRef: test.path.String(),
ShortRef: "short", ShortRef: "short",
ItemRef: test.path.Item(),
} }
pvs, err := test.cat.pathValues(test.path, ent) pvs, err := test.cat.pathValues(test.path, ent, Config{})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, test.expect, pvs) assert.Equal(t, test.expect, pvs)
}) })

View File

@ -60,6 +60,7 @@ func (mc mockCategorizer) isLeaf() bool {
func (mc mockCategorizer) pathValues( func (mc mockCategorizer) pathValues(
repo path.Path, repo path.Path,
ent details.DetailsEntry, ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) { ) (map[categorizer][]string, error) {
return map[categorizer][]string{ return map[categorizer][]string{
rootCatStub: {"root"}, rootCatStub: {"root"},

View File

@ -3,12 +3,10 @@ package selectors
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/filters"
@ -394,6 +392,7 @@ func (c oneDriveCategory) isLeaf() bool {
func (c oneDriveCategory) pathValues( func (c oneDriveCategory) pathValues(
repo path.Path, repo path.Path,
ent details.DetailsEntry, ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) { ) (map[categorizer][]string, error) {
if ent.OneDrive == nil { if ent.OneDrive == nil {
return nil, clues.New("no OneDrive ItemInfo in details") return nil, clues.New("no OneDrive ItemInfo in details")
@ -402,11 +401,18 @@ func (c oneDriveCategory) pathValues(
// Ignore `drives/<driveID>/root:` for folder comparison // Ignore `drives/<driveID>/root:` for folder comparison
rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String() rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String()
itemID := strings.TrimSuffix(repo.Item(), metadata.DataFileSuffix) item := ent.ItemRef
if len(item) == 0 {
item = repo.Item()
}
if cfg.OnlyMatchItemNames {
item = ent.ItemInfo.OneDrive.ItemName
}
result := map[categorizer][]string{ result := map[categorizer][]string{
OneDriveFolder: {rFld}, OneDriveFolder: {rFld},
OneDriveItem: {ent.OneDrive.ItemName, ent.ShortRef, itemID}, OneDriveItem: {item, ent.ShortRef},
} }
if len(ent.LocationRef) > 0 { if len(ent.LocationRef) > 0 {

View File

@ -173,6 +173,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
Entries: []details.DetailsEntry{ Entries: []details.DetailsEntry{
{ {
RepoRef: file, RepoRef: file,
ItemRef: "file",
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -182,6 +183,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
}, },
{ {
RepoRef: file2, RepoRef: file2,
ItemRef: "file2",
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -191,6 +193,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
}, },
{ {
RepoRef: file3, RepoRef: file3,
// item ref intentionally blank to assert fallback case
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -211,36 +214,69 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
deets *details.Details deets *details.Details
makeSelector func() *OneDriveRestore makeSelector func() *OneDriveRestore
expect []string expect []string
cfg Config
}{ }{
{ {
"all", name: "all",
deets, deets: deets,
func() *OneDriveRestore { makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any()) odr := NewOneDriveRestore(Any())
odr.Include(odr.AllData()) odr.Include(odr.AllData())
return odr return odr
}, },
arr(file, file2, file3), expect: arr(file, file2, file3),
}, },
{ {
"only match file", name: "only match file",
deets, deets: deets,
func() *OneDriveRestore { makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"file2"}))
return odr
},
expect: arr(file2),
},
{
name: "id doesn't match name",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"file2"}))
return odr
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "only match file name",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any()) odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"fileName2"})) odr.Include(odr.Items(Any(), []string{"fileName2"}))
return odr return odr
}, },
arr(file2), expect: arr(file2),
cfg: Config{OnlyMatchItemNames: true},
}, },
{ {
"only match folder", name: "name doesn't match id",
deets, deets: deets,
func() *OneDriveRestore { makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore(Any())
odr.Include(odr.Items(Any(), []string{"fileName2"}))
return odr
},
expect: []string{},
},
{
name: "only match folder",
deets: deets,
makeSelector: func() *OneDriveRestore {
odr := NewOneDriveRestore([]string{"uid"}) odr := NewOneDriveRestore([]string{"uid"})
odr.Include(odr.Folders([]string{"folderA/folderB", "folderA/folderC"})) odr.Include(odr.Folders([]string{"folderA/folderB", "folderA/folderC"}))
return odr return odr
}, },
arr(file, file2), expect: arr(file, file2),
}, },
} }
for _, test := range table { for _, test := range table {
@ -251,6 +287,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
defer flush() defer flush()
sel := test.makeSelector() sel := test.makeSelector()
sel.Configure(test.cfg)
results := sel.Reduce(ctx, test.deets, fault.New(true)) results := sel.Reduce(ctx, test.deets, fault.New(true))
paths := results.Paths() paths := results.Paths()
assert.Equal(t, test.expect, paths) assert.Equal(t, test.expect, paths)
@ -262,30 +299,68 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
t := suite.T() t := suite.T()
fileName := "file" fileName := "file"
fileID := fileName + "-id"
shortRef := "short" shortRef := "short"
elems := []string{"drive", "driveID", "root:", "dir1", "dir2", fileName + "-id"} elems := []string{"drive", "driveID", "root:", "dir1", "dir2", fileID}
filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...) filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
expected := map[categorizer][]string{ table := []struct {
OneDriveFolder: {"dir1/dir2"}, name string
OneDriveItem: {fileName, shortRef, fileName + "-id"}, pathElems []string
} expected map[categorizer][]string
cfg Config
ent := details.DetailsEntry{ }{
RepoRef: filePath.String(), {
ShortRef: shortRef, name: "items",
ItemInfo: details.ItemInfo{ pathElems: elems,
OneDrive: &details.OneDriveInfo{ expected: map[categorizer][]string{
ItemName: fileName, OneDriveFolder: {"dir1/dir2"},
OneDriveItem: {fileID, shortRef},
}, },
cfg: Config{},
},
{
name: "items w/ name",
pathElems: elems,
expected: map[categorizer][]string{
OneDriveFolder: {"dir1/dir2"},
OneDriveItem: {fileName, shortRef},
},
cfg: Config{OnlyMatchItemNames: true},
}, },
} }
r, err := OneDriveItem.pathValues(filePath, ent) for _, test := range table {
require.NoError(t, err) suite.Run(test.name, func() {
assert.Equal(t, expected, r) t := suite.T()
itemPath, err := path.Build(
"tenant",
"site",
path.OneDriveService,
path.FilesCategory,
true,
test.pathElems...)
require.NoError(t, err, clues.ToCore(err))
ent := details.DetailsEntry{
RepoRef: filePath.String(),
ShortRef: shortRef,
ItemRef: fileID,
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemName: fileName,
},
},
}
pv, err := OneDriveItem.pathValues(itemPath, ent, test.cfg)
require.NoError(t, err)
assert.Equal(t, test.expected, pv)
})
}
} }
func (suite *OneDriveSelectorSuite) TestOneDriveScope_MatchesInfo() { func (suite *OneDriveSelectorSuite) TestOneDriveScope_MatchesInfo() {

View File

@ -89,7 +89,7 @@ type (
// folderCat: folder, // folderCat: folder,
// itemCat: itemID, // itemCat: itemID,
// } // }
pathValues(path.Path, details.DetailsEntry) (map[categorizer][]string, error) pathValues(path.Path, details.DetailsEntry, Config) (map[categorizer][]string, error)
// pathKeys produces a list of categorizers that can be used as keys in the pathValues // pathKeys produces a list of categorizers that can be used as keys in the pathValues
// map. The combination of the two funcs generically interprets the context of the // map. The combination of the two funcs generically interprets the context of the
@ -389,7 +389,7 @@ func reduce[T scopeT, C categoryT](
continue continue
} }
pv, err := dc.pathValues(repoPath, *ent) pv, err := dc.pathValues(repoPath, *ent, s.Cfg)
if err != nil { if err != nil {
el.AddRecoverable(clues.Wrap(err, "getting path values").WithClues(ictx)) el.AddRecoverable(clues.Wrap(err, "getting path values").WithClues(ictx))
continue continue

View File

@ -366,7 +366,7 @@ func (suite *SelectorScopesSuite) TestPasses() {
} }
) )
pvs, err := cat.pathValues(pth, entry) pvs, err := cat.pathValues(pth, entry, Config{})
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
for _, test := range reduceTestTable { for _, test := range reduceTestTable {

View File

@ -122,6 +122,16 @@ type Selector struct {
// A slice of inclusion scopes. Comparators must match either one of these, // A slice of inclusion scopes. Comparators must match either one of these,
// or all filters, to be included. // or all filters, to be included.
Includes []scope `json:"includes,omitempty"` Includes []scope `json:"includes,omitempty"`
Cfg Config `json:"cfg,omitempty"`
}
// Config defines broad-scale selector behavior.
type Config struct {
// OnlyMatchItemNames tells the reducer to ignore matching on itemRef values
// and other item IDs in favor of matching the item name. Normal behavior only
// matches on itemRefs.
OnlyMatchItemNames bool
} }
// helper for specific selector instance constructors. // helper for specific selector instance constructors.
@ -140,6 +150,11 @@ func newSelector(s service, resourceOwners []string) Selector {
} }
} }
// Configure sets the selector configuration.
func (s *Selector) Configure(cfg Config) {
s.Cfg = cfg
}
// DiscreteResourceOwners returns the list of individual resourceOwners used // DiscreteResourceOwners returns the list of individual resourceOwners used
// in the selector. // in the selector.
// TODO(rkeepers): remove in favor of split and s.DiscreteOwner // TODO(rkeepers): remove in favor of split and s.DiscreteOwner

View File

@ -3,12 +3,10 @@ package selectors
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/filters"
@ -519,10 +517,10 @@ func (c sharePointCategory) isLeaf() bool {
func (c sharePointCategory) pathValues( func (c sharePointCategory) pathValues(
repo path.Path, repo path.Path,
ent details.DetailsEntry, ent details.DetailsEntry,
cfg Config,
) (map[categorizer][]string, error) { ) (map[categorizer][]string, error) {
var ( var (
folderCat, itemCat categorizer folderCat, itemCat categorizer
itemName = repo.Item()
dropDriveFolderPrefix bool dropDriveFolderPrefix bool
itemID string itemID string
) )
@ -535,8 +533,6 @@ func (c sharePointCategory) pathValues(
dropDriveFolderPrefix = true dropDriveFolderPrefix = true
folderCat, itemCat = SharePointLibraryFolder, SharePointLibraryItem folderCat, itemCat = SharePointLibraryFolder, SharePointLibraryItem
itemID = strings.TrimSuffix(itemName, metadata.DataFileSuffix)
itemName = ent.SharePoint.ItemName
case SharePointList, SharePointListItem: case SharePointList, SharePointListItem:
folderCat, itemCat = SharePointList, SharePointListItem folderCat, itemCat = SharePointList, SharePointListItem
@ -554,9 +550,18 @@ func (c sharePointCategory) pathValues(
rFld = path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String() rFld = path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String()
} }
item := ent.ItemRef
if len(item) == 0 {
item = repo.Item()
}
if cfg.OnlyMatchItemNames {
item = ent.ItemInfo.SharePoint.ItemName
}
result := map[categorizer][]string{ result := map[categorizer][]string{
folderCat: {rFld}, folderCat: {rFld},
itemCat: {itemName, ent.ShortRef}, itemCat: {item, ent.ShortRef},
} }
if len(itemID) > 0 { if len(itemID) > 0 {

View File

@ -220,6 +220,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
Entries: []details.DetailsEntry{ Entries: []details.DetailsEntry{
{ {
RepoRef: item, RepoRef: item,
ItemRef: "item",
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary, ItemType: details.SharePointLibrary,
@ -229,6 +230,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
}, },
{ {
RepoRef: item2, RepoRef: item2,
// ItemRef intentionally blank to test fallback case
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary, ItemType: details.SharePointLibrary,
@ -238,6 +240,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
}, },
{ {
RepoRef: item3, RepoRef: item3,
ItemRef: "item3",
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary, ItemType: details.SharePointLibrary,
@ -247,6 +250,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
}, },
{ {
RepoRef: item4, RepoRef: item4,
ItemRef: "item4",
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointPage, ItemType: details.SharePointPage,
@ -256,6 +260,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
}, },
{ {
RepoRef: item5, RepoRef: item5,
// ItemRef intentionally blank to test fallback case
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemType: details.SharePointPage, ItemType: details.SharePointPage,
@ -276,6 +281,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
deets *details.Details deets *details.Details
makeSelector func() *SharePointRestore makeSelector func() *SharePointRestore
expect []string expect []string
cfg Config
}{ }{
{ {
name: "all", name: "all",
@ -290,12 +296,44 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
{ {
name: "only match item", name: "only match item",
deets: deets, deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"item2"}))
return odr
},
expect: arr(item2),
},
{
name: "id doesn't match name",
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"item2"}))
return odr
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "only match item name",
deets: deets,
makeSelector: func() *SharePointRestore { makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any()) odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"itemName2"})) odr.Include(odr.LibraryItems(Any(), []string{"itemName2"}))
return odr return odr
}, },
expect: arr(item2), expect: arr(item2),
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "name doesn't match",
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore(Any())
odr.Include(odr.LibraryItems(Any(), []string{"itemName2"}))
return odr
},
expect: []string{},
}, },
{ {
name: "only match folder", name: "only match folder",
@ -326,6 +364,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
defer flush() defer flush()
sel := test.makeSelector() sel := test.makeSelector()
sel.Configure(test.cfg)
results := sel.Reduce(ctx, test.deets, fault.New(true)) results := sel.Reduce(ctx, test.deets, fault.New(true))
paths := results.Paths() paths := results.Paths()
assert.Equal(t, test.expect, paths) assert.Equal(t, test.expect, paths)
@ -336,9 +375,10 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
var ( var (
itemName = "item" itemName = "item"
itemID = "item-id"
shortRef = "short" shortRef = "short"
driveElems = []string{"drive", "drive!id", "root:", "dir1", "dir2", itemName + "-id"} driveElems = []string{"drive", "drive!id", "root:", "dir1", "dir2", itemID}
elems = []string{"dir1", "dir2", itemName + "-id"} elems = []string{"dir1", "dir2", itemID}
) )
table := []struct { table := []struct {
@ -346,6 +386,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
sc sharePointCategory sc sharePointCategory
pathElems []string pathElems []string
expected map[categorizer][]string expected map[categorizer][]string
cfg Config
}{ }{
{ {
name: "SharePoint Libraries", name: "SharePoint Libraries",
@ -353,8 +394,19 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
pathElems: driveElems, pathElems: driveElems,
expected: map[categorizer][]string{ expected: map[categorizer][]string{
SharePointLibraryFolder: {"dir1/dir2"}, SharePointLibraryFolder: {"dir1/dir2"},
SharePointLibraryItem: {itemName, shortRef, itemName + "-id"}, SharePointLibraryItem: {itemID, shortRef},
}, },
cfg: Config{},
},
{
name: "SharePoint Libraries w/ name",
sc: SharePointLibraryItem,
pathElems: driveElems,
expected: map[categorizer][]string{
SharePointLibraryFolder: {"dir1/dir2"},
SharePointLibraryItem: {itemName, shortRef},
},
cfg: Config{OnlyMatchItemNames: true},
}, },
{ {
name: "SharePoint Lists", name: "SharePoint Lists",
@ -362,8 +414,9 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
pathElems: elems, pathElems: elems,
expected: map[categorizer][]string{ expected: map[categorizer][]string{
SharePointList: {"dir1/dir2"}, SharePointList: {"dir1/dir2"},
SharePointListItem: {"item-id", shortRef}, SharePointListItem: {itemID, shortRef},
}, },
cfg: Config{},
}, },
} }
@ -383,6 +436,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
ent := details.DetailsEntry{ ent := details.DetailsEntry{
RepoRef: itemPath.String(), RepoRef: itemPath.String(),
ShortRef: shortRef, ShortRef: shortRef,
ItemRef: itemPath.Item(),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{ SharePoint: &details.SharePointInfo{
ItemName: itemName, ItemName: itemName,
@ -390,7 +444,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
}, },
} }
pv, err := test.sc.pathValues(itemPath, ent) pv, err := test.sc.pathValues(itemPath, ent, test.cfg)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, test.expected, pv) assert.Equal(t, test.expected, pv)
}) })