use selector owners, not scope owners (#1890)

## Description

Migrates code away from pulling the resource
owner from each scope, and instead usees the
selector as the canon identifier of the resource
owner.

## Does this PR need a docs update or release note?

- [x]  No 

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1617

## Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-01-04 12:39:25 -07:00 committed by GitHub
parent 84db56cc70
commit 0d0a7516f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 242 additions and 338 deletions

View File

@ -282,33 +282,30 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
bIDs []model.StableID bIDs []model.StableID
) )
for _, sel := range sel.SplitByResourceOwner(users) { for _, discSel := range sel.SplitByResourceOwner(users) {
// TODO: pass in entire selector, not individual scopes bo, err := r.NewBackup(ctx, discSel.Selector)
for _, scope := range sel.Scopes() { if err != nil {
bo, err := r.NewBackup(ctx, sel.Selector) errs = multierror.Append(errs, errors.Wrapf(
if err != nil { err,
errs = multierror.Append(errs, errors.Wrapf( "Failed to initialize Exchange backup for user %s",
err, discSel.DiscreteOwner,
"Failed to initialize Exchange backup for user %s", ))
scope.Get(selectors.ExchangeUser),
))
continue continue
}
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run Exchange backup for user %s",
scope.Get(selectors.ExchangeUser),
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
} }
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run Exchange backup for user %s",
discSel.DiscreteOwner,
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
} }
bups, err := r.Backups(ctx, bIDs) bups, err := r.Backups(ctx, bIDs)

View File

@ -204,33 +204,30 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
bIDs []model.StableID bIDs []model.StableID
) )
for _, sel := range sel.SplitByResourceOwner(users) { for _, discSel := range sel.SplitByResourceOwner(users) {
// TODO: pass in entire selector, not individual scopes bo, err := r.NewBackup(ctx, discSel.Selector)
for _, scope := range sel.Scopes() { if err != nil {
bo, err := r.NewBackup(ctx, sel.Selector) errs = multierror.Append(errs, errors.Wrapf(
if err != nil { err,
errs = multierror.Append(errs, errors.Wrapf( "Failed to initialize OneDrive backup for user %s",
err, discSel.DiscreteOwner,
"Failed to initialize OneDrive backup for user %s", ))
scope.Get(selectors.OneDriveUser),
))
continue continue
}
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run OneDrive backup for user %s",
scope.Get(selectors.OneDriveUser),
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
} }
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run OneDrive backup for user %s",
discSel.DiscreteOwner,
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
} }
bups, err := r.Backups(ctx, bIDs) bups, err := r.Backups(ctx, bIDs)

View File

@ -210,33 +210,30 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
bIDs []model.StableID bIDs []model.StableID
) )
for _, sel := range sel.SplitByResourceOwner(gc.GetSiteIDs()) { for _, discSel := range sel.SplitByResourceOwner(gc.GetSiteIDs()) {
// TODO: pass in entire selector, not individual scopes bo, err := r.NewBackup(ctx, discSel.Selector)
for _, scope := range sel.Scopes() { if err != nil {
bo, err := r.NewBackup(ctx, sel.Selector) errs = multierror.Append(errs, errors.Wrapf(
if err != nil { err,
errs = multierror.Append(errs, errors.Wrapf( "Failed to initialize SharePoint backup for site %s",
err, discSel.DiscreteOwner,
"Failed to initialize SharePoint backup for site %s", ))
scope.Get(selectors.SharePointSite),
))
continue continue
}
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run SharePoint backup for site %s",
scope.Get(selectors.SharePointSite),
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
} }
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run SharePoint backup for site %s",
discSel.DiscreteOwner,
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
} }
bups, err := r.Backups(ctx, bIDs) bups, err := r.Backups(ctx, bIDs)

View File

@ -47,7 +47,6 @@ func (gc *GraphConnector) DataCollections(
ctx, ctx,
sels, sels,
metadata, metadata,
gc.GetUsers(),
gc.credentials, gc.credentials,
// gc.Service, // gc.Service,
gc.UpdateStatus, gc.UpdateStatus,
@ -154,31 +153,29 @@ func (gc *GraphConnector) OneDriveDataCollections(
} }
var ( var (
scopes = odb.DiscreteScopes([]string{selector.DiscreteOwner}) user = selector.DiscreteOwner
collections = []data.Collection{} collections = []data.Collection{}
errs error errs error
) )
// for each scope that includes oneDrive items, get all // for each scope that includes oneDrive items, get all
for _, scope := range scopes { for _, scope := range odb.Scopes() {
for _, user := range scope.Get(selectors.OneDriveUser) { logger.Ctx(ctx).With("user", user).Debug("Creating OneDrive collections")
logger.Ctx(ctx).With("user", user).Debug("Creating OneDrive collections")
odcs, err := onedrive.NewCollections( odcs, err := onedrive.NewCollections(
gc.credentials.AzureTenantID, gc.credentials.AzureTenantID,
user, user,
onedrive.OneDriveSource, onedrive.OneDriveSource,
odFolderMatcher{scope}, odFolderMatcher{scope},
gc.Service, gc.Service,
gc.UpdateStatus, gc.UpdateStatus,
ctrlOpts, ctrlOpts,
).Get(ctx) ).Get(ctx)
if err != nil { if err != nil {
return nil, support.WrapAndAppend(user, err, errs) return nil, support.WrapAndAppend(user, err, errs)
}
collections = append(collections, odcs...)
} }
collections = append(collections, odcs...)
} }
for range collections { for range collections {

View File

@ -71,7 +71,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
getSelector: func(t *testing.T) selectors.Selector { getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup(selUsers) sel := selectors.NewExchangeBackup(selUsers)
sel.Include(sel.MailFolders(selUsers, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) sel.Include(sel.MailFolders(selUsers, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
sel.DiscreteOwner = suite.user
return sel.Selector return sel.Selector
}, },
}, },
@ -79,11 +79,8 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
name: suite.user + " Contacts", name: suite.user + " Contacts",
getSelector: func(t *testing.T) selectors.Selector { getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup(selUsers) sel := selectors.NewExchangeBackup(selUsers)
sel.Include(sel.ContactFolders( sel.Include(sel.ContactFolders(selUsers, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch()))
selUsers, sel.DiscreteOwner = suite.user
[]string{exchange.DefaultContactFolder},
selectors.PrefixMatch()))
return sel.Selector return sel.Selector
}, },
}, },
@ -91,12 +88,8 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
// name: suite.user + " Events", // name: suite.user + " Events",
// getSelector: func(t *testing.T) selectors.Selector { // getSelector: func(t *testing.T) selectors.Selector {
// sel := selectors.NewExchangeBackup(selUsers) // sel := selectors.NewExchangeBackup(selUsers)
// sel.Include(sel.EventCalendars( // sel.Include(sel.EventCalendars(selUsers, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
// selUsers, // sel.DiscreteOwner = suite.user
// []string{exchange.DefaultCalendar},
// selectors.PrefixMatch(),
// ))
// return sel.Selector // return sel.Selector
// }, // },
// }, // },
@ -108,7 +101,6 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
ctx, ctx,
test.getSelector(t), test.getSelector(t),
nil, nil,
[]string{suite.user},
connector.credentials, connector.credentials,
connector.UpdateStatus, connector.UpdateStatus,
control.Options{}) control.Options{})
@ -199,6 +191,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
sel := selectors.NewSharePointBackup(selSites) sel := selectors.NewSharePointBackup(selSites)
sel.Include(sel.Libraries(selSites, selectors.Any())) sel.Include(sel.Libraries(selSites, selectors.Any()))
sel.DiscreteOwner = suite.site sel.DiscreteOwner = suite.site
return sel.Selector return sel.Selector
}, },
}, },
@ -209,6 +202,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
sel := selectors.NewSharePointBackup(selSites) sel := selectors.NewSharePointBackup(selSites)
sel.Include(sel.Lists(selSites, selectors.Any())) sel.Include(sel.Lists(selSites, selectors.Any()))
sel.DiscreteOwner = suite.site sel.DiscreteOwner = suite.site
return sel.Selector return sel.Selector
}, },
}, },

View File

@ -163,7 +163,6 @@ func DataCollections(
ctx context.Context, ctx context.Context,
selector selectors.Selector, selector selectors.Selector,
metadata []data.Collection, metadata []data.Collection,
userPNs []string,
acct account.M365Config, acct account.M365Config,
su support.StatusUpdater, su support.StatusUpdater,
ctrlOpts control.Options, ctrlOpts control.Options,
@ -174,7 +173,8 @@ func DataCollections(
} }
var ( var (
scopes = eb.DiscreteScopes(userPNs) user = selector.DiscreteOwner
scopes = eb.DiscreteScopes([]string{user})
collections = []data.Collection{} collections = []data.Collection{}
errs error errs error
) )
@ -190,13 +190,13 @@ func DataCollections(
dcs, err := createCollections( dcs, err := createCollections(
ctx, ctx,
acct, acct,
user,
scope, scope,
dps, dps,
ctrlOpts, ctrlOpts,
su) su)
if err != nil { if err != nil {
user := scope.Get(selectors.ExchangeUser) return nil, support.WrapAndAppend(user, err, errs)
return nil, support.WrapAndAppend(user[0], err, errs)
} }
collections = append(collections, dcs...) collections = append(collections, dcs...)
@ -211,6 +211,7 @@ func DataCollections(
func createCollections( func createCollections(
ctx context.Context, ctx context.Context,
acct account.M365Config, acct account.M365Config,
user string,
scope selectors.ExchangeScope, scope selectors.ExchangeScope,
dps DeltaPaths, dps DeltaPaths,
ctrlOpts control.Options, ctrlOpts control.Options,
@ -218,48 +219,45 @@ func createCollections(
) ([]data.Collection, error) { ) ([]data.Collection, error) {
var ( var (
errs *multierror.Error errs *multierror.Error
users = scope.Get(selectors.ExchangeUser)
allCollections = make([]data.Collection, 0) allCollections = make([]data.Collection, 0)
) )
// Create collection of ExchangeDataCollection // Create collection of ExchangeDataCollection
for _, user := range users { collections := make(map[string]data.Collection)
collections := make(map[string]data.Collection)
qp := graph.QueryParams{ qp := graph.QueryParams{
Category: scope.Category().PathType(), Category: scope.Category().PathType(),
ResourceOwner: user, ResourceOwner: user,
Credentials: acct, Credentials: acct,
} }
foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, user)) foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, user))
defer closer() defer closer()
defer close(foldersComplete) defer close(foldersComplete)
resolver, err := PopulateExchangeContainerResolver(ctx, qp) resolver, err := PopulateExchangeContainerResolver(ctx, qp)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "getting folder cache") return nil, errors.Wrap(err, "getting folder cache")
} }
err = filterContainersAndFillCollections( err = filterContainersAndFillCollections(
ctx, ctx,
qp, qp,
collections, collections,
su, su,
resolver, resolver,
scope, scope,
dps, dps,
ctrlOpts) ctrlOpts)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "filling collections") return nil, errors.Wrap(err, "filling collections")
} }
foldersComplete <- struct{}{} foldersComplete <- struct{}{}
for _, coll := range collections { for _, coll := range collections {
allCollections = append(allCollections, coll) allCollections = append(allCollections, coll)
}
} }
return allCollections, errs.ErrorOrNil() return allCollections, errs.ErrorOrNil()

View File

@ -256,13 +256,12 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() {
}, },
} }
// gc := loadConnector(ctx, t, Users)
for _, test := range tests { for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
collections, err := createCollections( collections, err := createCollections(
ctx, ctx,
acct, acct,
userID,
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Options{}, control.Options{},
@ -324,6 +323,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
collections, err := createCollections( collections, err := createCollections(
ctx, ctx,
acct, acct,
userID,
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Options{}, control.Options{},
@ -351,6 +351,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
collections, err = createCollections( collections, err = createCollections(
ctx, ctx,
acct, acct,
userID,
test.scope, test.scope,
dps, dps,
control.Options{}, control.Options{},
@ -395,6 +396,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
collections, err := createCollections( collections, err := createCollections(
ctx, ctx,
acct, acct,
suite.user,
sel.Scopes()[0], sel.Scopes()[0],
DeltaPaths{}, DeltaPaths{},
control.Options{}, control.Options{},
@ -462,6 +464,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
edcs, err := createCollections( edcs, err := createCollections(
ctx, ctx,
acct, acct,
suite.user,
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Options{}, control.Options{},
@ -546,6 +549,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
collections, err := createCollections( collections, err := createCollections(
ctx, ctx,
acct, acct,
suite.user,
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Options{}, control.Options{},

View File

@ -10,6 +10,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
@ -777,14 +778,14 @@ func makeExchangeBackupSel(
dests []destAndCats, dests []destAndCats,
) selectors.Selector { ) selectors.Selector {
toInclude := [][]selectors.ExchangeScope{} toInclude := [][]selectors.ExchangeScope{}
resourceOwners := []string{} resourceOwners := map[string]struct{}{}
for _, d := range dests { for _, d := range dests {
for c := range d.cats { for c := range d.cats {
sel := selectors.NewExchangeBackup(nil) sel := selectors.NewExchangeBackup(nil)
builder := sel.MailFolders builder := sel.MailFolders
resourceOwners = append(resourceOwners, d.resourceOwner) resourceOwners[d.resourceOwner] = struct{}{}
switch c { switch c {
case path.ContactsCategory: case path.ContactsCategory:
@ -802,7 +803,7 @@ func makeExchangeBackupSel(
} }
} }
sel := selectors.NewExchangeBackup(resourceOwners) sel := selectors.NewExchangeBackup(maps.Keys(resourceOwners))
sel.Include(toInclude...) sel.Include(toInclude...)
return sel.Selector return sel.Selector
@ -938,20 +939,37 @@ func collectionsForInfo(
} }
//nolint:deadcode //nolint:deadcode
func getSelectorWith(service path.ServiceType) selectors.Selector { func getSelectorWith(
s := selectors.ServiceUnknown t *testing.T,
service path.ServiceType,
resourceOwners []string,
forRestore bool,
) selectors.Selector {
switch service { switch service {
case path.ExchangeService: case path.ExchangeService:
s = selectors.ServiceExchange if forRestore {
case path.OneDriveService: return selectors.NewExchangeRestore(resourceOwners).Selector
s = selectors.ServiceOneDrive }
case path.SharePointService:
s = selectors.ServiceSharePoint
}
return selectors.Selector{ return selectors.NewExchangeBackup(resourceOwners).Selector
Service: s,
case path.OneDriveService:
if forRestore {
return selectors.NewOneDriveRestore(resourceOwners).Selector
}
return selectors.NewOneDriveBackup(resourceOwners).Selector
case path.SharePointService:
if forRestore {
return selectors.NewSharePointRestore(resourceOwners).Selector
}
return selectors.NewSharePointBackup(resourceOwners).Selector
default:
require.FailNow(t, "unknown path service")
return selectors.Selector{}
} }
} }

View File

@ -319,7 +319,7 @@ func runRestoreBackupTest(
acct account.Account, acct account.Account,
test restoreBackupInfo, test restoreBackupInfo,
tenant string, tenant string,
users []string, resourceOwners []string,
) { ) {
var ( var (
collections []data.Collection collections []data.Collection
@ -332,32 +332,32 @@ func runRestoreBackupTest(
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
for _, user := range users { for _, owner := range resourceOwners {
numItems, userCollections, userExpectedData := collectionsForInfo( numItems, ownerCollections, userExpectedData := collectionsForInfo(
t, t,
test.service, test.service,
tenant, tenant,
user, owner,
dest, dest,
test.collections, test.collections,
) )
collections = append(collections, userCollections...) collections = append(collections, ownerCollections...)
totalItems += numItems totalItems += numItems
maps.Copy(expectedData, userExpectedData) maps.Copy(expectedData, userExpectedData)
} }
t.Logf( t.Logf(
"Restoring collections to %s for user(s) %v\n", "Restoring collections to %s for resourceOwners(s) %v\n",
dest.ContainerName, dest.ContainerName,
users, resourceOwners,
) )
start := time.Now() start := time.Now()
restoreGC := loadConnector(ctx, t, test.resource) restoreGC := loadConnector(ctx, t, test.resource)
restoreSel := getSelectorWith(test.service) restoreSel := getSelectorWith(t, test.service, resourceOwners, true)
deets, err := restoreGC.RestoreDataCollections( deets, err := restoreGC.RestoreDataCollections(
ctx, ctx,
acct, acct,
@ -386,10 +386,10 @@ func runRestoreBackupTest(
cats[c.category] = struct{}{} cats[c.category] = struct{}{}
} }
expectedDests := make([]destAndCats, 0, len(users)) expectedDests := make([]destAndCats, 0, len(resourceOwners))
for _, u := range users { for _, ro := range resourceOwners {
expectedDests = append(expectedDests, destAndCats{ expectedDests = append(expectedDests, destAndCats{
resourceOwner: u, resourceOwner: ro,
dest: dest.ContainerName, dest: dest.ContainerName,
cats: cats, cats: cats,
}) })
@ -809,7 +809,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
restoreSel := getSelectorWith(test.service) restoreSel := getSelectorWith(t, test.service, []string{suite.user}, true)
expectedDests := make([]destAndCats, 0, len(test.collections)) expectedDests := make([]destAndCats, 0, len(test.collections))
allItems := 0 allItems := 0
allExpectedData := map[string]map[string][]byte{} allExpectedData := map[string]map[string][]byte{}
@ -883,115 +883,3 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
}) })
} }
} }
func (suite *GraphConnectorIntegrationSuite) TestMultiuserRestoreAndBackup() {
bodyText := "This email has some text. However, all the text is on the same line."
subjectText := "Test message for restore"
users := []string{
suite.user,
tester.SecondaryM365UserID(suite.T()),
}
table := []restoreBackupInfo{
{
name: "Email",
service: path.ExchangeService,
resource: Users,
collections: []colInfo{
{
pathElements: []string{"Inbox"},
category: path.EmailCategory,
items: []itemInfo{
{
name: "someencodeditemID",
data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-1",
bodyText+" 1.",
bodyText+" 1.",
),
lookupKey: subjectText + "-1",
},
},
},
{
pathElements: []string{"Archive"},
category: path.EmailCategory,
items: []itemInfo{
{
name: "someencodeditemID2",
data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-2",
bodyText+" 2.",
bodyText+" 2.",
),
lookupKey: subjectText + "-2",
},
},
},
},
},
{
name: "Contacts",
service: path.ExchangeService,
resource: Users,
collections: []colInfo{
{
pathElements: []string{"Work"},
category: path.ContactsCategory,
items: []itemInfo{
{
name: "someencodeditemID",
data: mockconnector.GetMockContactBytes("Ghimley"),
lookupKey: "Ghimley",
},
},
},
{
pathElements: []string{"Personal"},
category: path.ContactsCategory,
items: []itemInfo{
{
name: "someencodeditemID2",
data: mockconnector.GetMockContactBytes("Irgot"),
lookupKey: "Irgot",
},
},
},
},
},
// {
// name: "Events",
// service: path.ExchangeService,
// collections: []colInfo{
// {
// pathElements: []string{"Work"},
// category: path.EventsCategory,
// items: []itemInfo{
// {
// name: "someencodeditemID",
// data: mockconnector.GetMockEventWithSubjectBytes("Ghimley"),
// lookupKey: "Ghimley",
// },
// },
// },
// {
// pathElements: []string{"Personal"},
// category: path.EventsCategory,
// items: []itemInfo{
// {
// name: "someencodeditemID2",
// data: mockconnector.GetMockEventWithSubjectBytes("Irgot"),
// lookupKey: "Irgot",
// },
// },
// },
// },
// },
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
runRestoreBackupTest(t, suite.acct, test, suite.connector.tenant, users)
})
}
}

View File

@ -37,55 +37,50 @@ func DataCollections(
} }
var ( var (
scopes = b.DiscreteScopes([]string{selector.DiscreteOwner}) site = b.DiscreteOwner
collections = []data.Collection{} collections = []data.Collection{}
errs error errs error
) )
for _, scope := range scopes { for _, scope := range b.Scopes() {
// due to DiscreteScopes(siteIDs), each range should only contain one site. foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf(
for _, site := range scope.Get(selectors.SharePointSite) { "∙ %s - %s:",
foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf( scope.Category().PathType(), site))
"∙ %s - %s:", defer closer()
scope.Category().PathType(), site)) defer close(foldersComplete)
defer closer()
defer close(foldersComplete)
var spcs []data.Collection var spcs []data.Collection
switch scope.Category().PathType() { switch scope.Category().PathType() {
case path.ListsCategory: case path.ListsCategory:
spcs, err = collectLists( spcs, err = collectLists(
ctx, ctx,
serv, serv,
tenantID, tenantID,
site, site,
scope, scope,
su, su,
ctrlOpts, ctrlOpts)
) if err != nil {
if err != nil { return nil, support.WrapAndAppend(site, err, errs)
return nil, support.WrapAndAppend(site, err, errs)
}
case path.LibrariesCategory:
spcs, err = collectLibraries(
ctx,
serv,
tenantID,
site,
scope,
su,
ctrlOpts)
if err != nil {
return nil, support.WrapAndAppend(site, err, errs)
}
} }
collections = append(collections, spcs...) case path.LibrariesCategory:
spcs, err = collectLibraries(
foldersComplete <- struct{}{} ctx,
serv,
tenantID,
site,
scope,
su,
ctrlOpts)
if err != nil {
return nil, support.WrapAndAppend(site, err, errs)
}
} }
collections = append(collections, spcs...)
foldersComplete <- struct{}{}
} }
return collections, errs return collections, errs

View File

@ -34,9 +34,10 @@ import (
type BackupOperation struct { type BackupOperation struct {
operation operation
Results BackupResults `json:"results"` ResourceOwner string `json:"resourceOwner"`
Selectors selectors.Selector `json:"selectors"` Results BackupResults `json:"results"`
Version string `json:"version"` Selectors selectors.Selector `json:"selectors"`
Version string `json:"version"`
account account.Account account account.Account
} }
@ -60,10 +61,11 @@ func NewBackupOperation(
bus events.Eventer, bus events.Eventer,
) (BackupOperation, error) { ) (BackupOperation, error) {
op := BackupOperation{ op := BackupOperation{
operation: newOperation(opts, bus, kw, sw), operation: newOperation(opts, bus, kw, sw),
Selectors: selector, ResourceOwner: selector.DiscreteOwner,
Version: "v0", Selectors: selector,
account: acct, Version: "v0",
account: acct,
} }
if err := op.validate(); err != nil { if err := op.validate(); err != nil {
return BackupOperation{}, err return BackupOperation{}, err
@ -73,6 +75,10 @@ func NewBackupOperation(
} }
func (op BackupOperation) validate() error { func (op BackupOperation) validate() error {
if len(op.ResourceOwner) == 0 {
return errors.New("backup requires a resource owner")
}
return op.operation.validate() return op.operation.validate()
} }
@ -336,9 +342,7 @@ func selectorToOwnersCats(sel selectors.Selector) *kopia.OwnersCats {
ServiceCats: map[string]kopia.ServiceCat{}, ServiceCats: map[string]kopia.ServiceCat{},
} }
for _, ro := range sel.DiscreteResourceOwners() { oc.ResourceOwners[sel.DiscreteOwner] = struct{}{}
oc.ResourceOwners[ro] = struct{}{}
}
pcs, err := sel.PathCategories() pcs, err := sel.PathCategories()
if err != nil { if err != nil {

View File

@ -484,7 +484,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
test.kw, test.kw,
test.sw, test.sw,
test.acct, test.acct,
selectors.Selector{}, selectors.Selector{DiscreteOwner: "test"},
evmock.NewBus()) evmock.NewBus())
test.errCheck(t, err) test.errCheck(t, err)
}) })
@ -516,6 +516,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() {
selector: func() *selectors.ExchangeBackup { selector: func() *selectors.ExchangeBackup {
sel := selectors.NewExchangeBackup(users) sel := selectors.NewExchangeBackup(users)
sel.Include(sel.MailFolders(users, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) sel.Include(sel.MailFolders(users, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
sel.DiscreteOwner = suite.user
return sel return sel
}, },
resourceOwner: suite.user, resourceOwner: suite.user,
@ -531,6 +533,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() {
users, users,
[]string{exchange.DefaultContactFolder}, []string{exchange.DefaultContactFolder},
selectors.PrefixMatch())) selectors.PrefixMatch()))
sel.DiscreteOwner = suite.user
return sel return sel
}, },
@ -544,6 +547,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() {
selector: func() *selectors.ExchangeBackup { selector: func() *selectors.ExchangeBackup {
sel := selectors.NewExchangeBackup(users) sel := selectors.NewExchangeBackup(users)
sel.Include(sel.EventCalendars(users, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())) sel.Include(sel.EventCalendars(users, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
sel.DiscreteOwner = suite.user
return sel return sel
}, },
resourceOwner: suite.user, resourceOwner: suite.user,
@ -876,16 +881,20 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
switch category { switch category {
case path.EmailCategory: case path.EmailCategory:
cmf := cli.MailFoldersById(containerID) cmf := cli.MailFoldersById(containerID)
body, err := cmf.Get(ctx, nil) body, err := cmf.Get(ctx, nil)
require.NoError(t, err, "getting mail folder") require.NoError(t, err, "getting mail folder")
body.SetDisplayName(&containerRename) body.SetDisplayName(&containerRename)
_, err = cmf.Patch(ctx, body, nil) _, err = cmf.Patch(ctx, body, nil)
require.NoError(t, err, "updating mail folder name") require.NoError(t, err, "updating mail folder name")
case path.ContactsCategory: case path.ContactsCategory:
ccf := cli.ContactFoldersById(containerID) ccf := cli.ContactFoldersById(containerID)
body, err := ccf.Get(ctx, nil) body, err := ccf.Get(ctx, nil)
require.NoError(t, err, "getting contact folder") require.NoError(t, err, "getting contact folder")
body.SetDisplayName(&containerRename) body.SetDisplayName(&containerRename)
_, err = ccf.Patch(ctx, body, nil) _, err = ccf.Patch(ctx, body, nil)
require.NoError(t, err, "updating contact folder name") require.NoError(t, err, "updating contact folder name")

View File

@ -364,7 +364,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
kw, kw,
sw, sw,
acct, acct,
selectors.Selector{}, selectors.Selector{DiscreteOwner: "test"},
evmock.NewBus()) evmock.NewBus())
require.NoError(t, err) require.NoError(t, err)
test.expectErr(t, op.persistResults(now, &test.stats)) test.expectErr(t, op.persistResults(now, &test.stats))

View File

@ -98,7 +98,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
sw, sw,
acct, acct,
"foo", "foo",
selectors.Selector{}, selectors.Selector{DiscreteOwner: "test"},
dest, dest,
evmock.NewBus()) evmock.NewBus())
require.NoError(t, err) require.NoError(t, err)
@ -250,7 +250,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
test.sw, test.sw,
test.acct, test.acct,
"backup-id", "backup-id",
selectors.Selector{}, selectors.Selector{DiscreteOwner: "test"},
dest, dest,
evmock.NewBus()) evmock.NewBus())
test.errCheck(t, err) test.errCheck(t, err)

View File

@ -193,7 +193,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
r, err := repository.Initialize(ctx, acct, st, control.Options{}) r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err) require.NoError(t, err)
bo, err := r.NewBackup(ctx, selectors.Selector{}) bo, err := r.NewBackup(ctx, selectors.Selector{DiscreteOwner: "test"})
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, bo) require.NotNil(t, bo)
} }
@ -213,7 +213,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
r, err := repository.Initialize(ctx, acct, st, control.Options{}) r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err) require.NoError(t, err)
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}, dest) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, dest)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, ro) require.NotNil(t, ro)
} }

View File

@ -113,9 +113,15 @@ type Selector struct {
// helper for specific selector instance constructors. // helper for specific selector instance constructors.
func newSelector(s service, resourceOwners []string) Selector { func newSelector(s service, resourceOwners []string) Selector {
var owner string
if len(resourceOwners) == 1 {
owner = resourceOwners[0]
}
return Selector{ return Selector{
Service: s, Service: s,
ResourceOwners: filterize(scopeConfig{}, resourceOwners...), ResourceOwners: filterize(scopeConfig{}, resourceOwners...),
DiscreteOwner: owner,
Excludes: []scope{}, Excludes: []scope{},
Includes: []scope{}, Includes: []scope{},
} }