Compare commits

...

11 Commits

Author SHA1 Message Date
ryanfkeepers
b00559acc8 display the backup owner name when listing 2023-03-30 13:33:50 -06:00
ryanfkeepers
ca2fbda260 match sites with a url path suffix
Adds suffix matching to the site owner id/name
lookup process.  This allows consumers to query
for a site by only its path (ex: "/sites/foo") and still
end up with a well formed id, name tuple.  One
gotcha is that this requires the lookup maps to
be populated.  If a lookup map is not passed in
with a suffix matcher, the fallback lookup will fail.
2023-03-29 17:14:47 -06:00
ryanfkeepers
702e25bcda rebase fixes 2023-03-29 17:13:51 -06:00
ryanfkeepers
9bb97f0ebd look up users and sites by id or name
Adds a lookup step to graph connector to find
an owner's id and name given some identifier.
The identifier, for either sites or users, can be a
well formed id or name.
2023-03-29 17:13:51 -06:00
ryanfkeepers
3531b3c84c move sharepoint onto map-based lookup
Refactors the sharepoint site lookup to use the
id-to-name maps.  This has a momentary regression
that will get solved in the next PR: we no longer
match on weburl suffixes, and instead require
a complete match.

Also, migrates the sharepoint lookup code out
of GC and into Discovery.
2023-03-29 17:12:49 -06:00
ryanfkeepers
cdbf3910e8 introduce id-name lookup maps
Adds two maps to resource-owner handling:
id-to-name and name-to-id.  Expectation is that
these maps will either get populated by a caller
as a pre-process before initializing the gc client,
or gc will (later pr) be able to look up the owner
and populate those maps itself.  The maps are
used to set the selector id and name for iface
compliance.  Only supported by exchange in this PR.
2023-03-29 17:09:53 -06:00
ryanfkeepers
547b8c767e introduce idnamer
Adds a common interface: idNamer, which is used
to pass around tuples of an id and a name for some
resource.  Also adds compliance to this iface in
selectors, where a selector's ID and Name are the
DiscreteOwner values.
2023-03-29 17:08:19 -06:00
ryanfkeepers
945e4ecbd0 fix tests, missing producer 2023-03-29 16:45:08 -06:00
ryanfkeepers
3da8bf86a4 rebase fixes 2023-03-29 16:45:08 -06:00
ryanfkeepers
b5527e9ef2 fix circular ref from prior change 2023-03-29 16:45:08 -06:00
ryanfkeepers
feddd9b183 replace graphConnector with interface
Replaces the operations graphConnector reference
with an interface.  Restore and Backups have
separate, unique interfaces.
2023-03-29 16:44:15 -06:00
48 changed files with 1616 additions and 716 deletions

View File

@ -195,6 +195,8 @@ func runBackups(
r repository.Repository,
serviceName, resourceOwnerType string,
selectorSet []selectors.Selector,
resourceOwnersIDToName map[string]string,
resourceOwnersNameToID map[string]string,
) error {
var (
bIDs []model.StableID
@ -204,21 +206,21 @@ func runBackups(
for _, discSel := range selectorSet {
var (
owner = discSel.DiscreteOwner
bctx = clues.Add(ctx, "resource_owner", owner)
ictx = clues.Add(ctx, "resource_owner", owner)
)
bo, err := r.NewBackup(bctx, discSel)
bo, err := r.NewBackup(ictx, discSel, resourceOwnersIDToName, resourceOwnersNameToID)
if err != nil {
errs = append(errs, clues.Wrap(err, owner).WithClues(bctx))
Errf(bctx, "%v\n", err)
errs = append(errs, clues.Wrap(err, owner).WithClues(ictx))
Errf(ictx, "%v\n", err)
continue
}
err = bo.Run(bctx)
err = bo.Run(ictx)
if err != nil {
errs = append(errs, clues.Wrap(err, owner).WithClues(bctx))
Errf(bctx, "%v\n", err)
errs = append(errs, clues.Wrap(err, owner).WithClues(ictx))
Errf(ictx, "%v\n", err)
continue
}

View File

@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print"
@ -164,14 +165,14 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
// TODO: log/print recoverable errors
errs := fault.New(false)
users, err := m365.UserPNs(ctx, *acct, errs)
idToPN, pnToID, err := m365.UsersMap(ctx, *acct, errs)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 user(s)"))
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
}
selectorSet := []selectors.Selector{}
for _, discSel := range sel.SplitByResourceOwner(users) {
for _, discSel := range sel.SplitByResourceOwner(maps.Keys(pnToID)) {
selectorSet = append(selectorSet, discSel.Selector)
}
@ -180,7 +181,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
r,
"Exchange", "user",
selectorSet,
)
idToPN, pnToID)
}
func exchangeBackupCreateSelectors(userIDs, cats []string) *selectors.ExchangeBackup {

View File

@ -300,7 +300,11 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
suite.backupOps = make(map[path.CategoryType]string)
users := []string{suite.m365UserID}
var (
users = []string{suite.m365UserID}
idToName = map[string]string{suite.m365UserID: "todo-name-" + suite.m365UserID}
nameToID = map[string]string{"todo-name-" + suite.m365UserID: suite.m365UserID}
)
for _, set := range backupDataSets {
var (
@ -321,7 +325,7 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
sel.Include(scopes)
bop, err := suite.repo.NewBackup(ctx, sel.Selector)
bop, err := suite.repo.NewBackup(ctx, sel.Selector, idToName, nameToID)
require.NoError(t, err, clues.ToCore(err))
err = bop.Run(ctx)
@ -546,7 +550,7 @@ func (suite *BackupDeleteExchangeE2ESuite) SetupSuite() {
sel := selectors.NewExchangeBackup(users)
sel.Include(sel.MailFolders([]string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector, nil, nil)
require.NoError(t, err, clues.ToCore(err))
err = suite.backupOp.Run(ctx)

View File

@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print"
@ -148,14 +149,14 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
// TODO: log/print recoverable errors
errs := fault.New(false)
users, err := m365.UserPNs(ctx, *acct, errs)
idToName, nameToID, err := m365.UsersMap(ctx, *acct, errs)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
}
selectorSet := []selectors.Selector{}
for _, discSel := range sel.SplitByResourceOwner(users) {
for _, discSel := range sel.SplitByResourceOwner(maps.Keys(idToName)) {
selectorSet = append(selectorSet, discSel.Selector)
}
@ -164,7 +165,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
r,
"OneDrive", "user",
selectorSet,
)
idToName, nameToID)
}
func validateOneDriveBackupCreateFlags(users []string) error {

View File

@ -205,14 +205,18 @@ func (suite *BackupDeleteOneDriveE2ESuite) SetupSuite() {
})
require.NoError(t, err, clues.ToCore(err))
m365UserID := tester.M365UserID(t)
users := []string{m365UserID}
var (
m365UserID = tester.M365UserID(t)
users = []string{m365UserID}
idToName = map[string]string{m365UserID: "todo-name-" + m365UserID}
nameToID = map[string]string{"todo-name-" + m365UserID: m365UserID}
)
// some tests require an existing backup
sel := selectors.NewOneDriveBackup(users)
sel.Include(sel.Folders(selectors.Any()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector, idToName, nameToID)
require.NoError(t, err, clues.ToCore(err))
err = suite.backupOp.Run(ctx)

View File

@ -7,18 +7,20 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
)
// ------------------------------------------------------------------------------------------------
@ -154,19 +156,19 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
// TODO: log/print recoverable errors
errs := fault.New(false)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), *acct, connector.Sites, errs)
idToURL, urlToID, err := m365.SitesMap(ctx, *acct, errs)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to connect to Microsoft APIs"))
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 sites"))
}
sel, err := sharePointBackupCreateSelectors(ctx, utils.SiteID, utils.WebURL, utils.CategoryData, gc)
sel, err := sharePointBackupCreateSelectors(ctx, urlToID, utils.SiteID, utils.WebURL, utils.CategoryData)
if err != nil {
return Only(ctx, clues.Wrap(err, "Retrieving up sharepoint sites by ID and URL"))
}
selectorSet := []selectors.Selector{}
for _, discSel := range sel.SplitByResourceOwner(gc.GetSiteIDs()) {
for _, discSel := range sel.SplitByResourceOwner(maps.Keys(idToURL)) {
selectorSet = append(selectorSet, discSel.Selector)
}
@ -175,7 +177,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
r,
"SharePoint", "site",
selectorSet,
)
idToURL, urlToID)
}
func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error {
@ -201,44 +203,30 @@ func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error {
// TODO: users might specify a data type, this only supports AllData().
func sharePointBackupCreateSelectors(
ctx context.Context,
urlToID map[string]string,
sites, weburls, cats []string,
gc *connector.GraphConnector,
) (*selectors.SharePointBackup, error) {
if len(sites) == 0 && len(weburls) == 0 {
return selectors.NewSharePointBackup(selectors.None()), nil
}
for _, site := range sites {
if site == utils.Wildcard {
return includeAllSitesWithCategories(cats), nil
}
if filters.PathContains(sites).Compare(utils.Wildcard) {
return includeAllSitesWithCategories(urlToID, cats), nil
}
for _, wURL := range weburls {
if wURL == utils.Wildcard {
return includeAllSitesWithCategories(cats), nil
}
if filters.PathContains(weburls).Compare(utils.Wildcard) {
return includeAllSitesWithCategories(urlToID, cats), nil
}
// TODO: log/print recoverable errors
errs := fault.New(false)
union, err := gc.UnionSiteIDsAndWebURLs(ctx, sites, weburls, errs)
if err != nil {
return nil, err
}
sel := selectors.NewSharePointBackup(union)
sel := selectors.NewSharePointBackup(append(slices.Clone(sites), weburls...))
return addCategories(sel, cats), nil
}
func includeAllSitesWithCategories(categories []string) *selectors.SharePointBackup {
sel := addCategories(
selectors.NewSharePointBackup(selectors.Any()),
func includeAllSitesWithCategories(urlToID map[string]string, categories []string) *selectors.SharePointBackup {
return addCategories(
selectors.NewSharePointBackup(maps.Values(urlToID)),
categories)
return sel
}
func addCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {

View File

@ -156,14 +156,18 @@ func (suite *BackupDeleteSharePointE2ESuite) SetupSuite() {
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err, clues.ToCore(err))
m365SiteID := tester.M365SiteID(t)
sites := []string{m365SiteID}
var (
m365SiteID = tester.M365SiteID(t)
sites = []string{m365SiteID}
idToName = map[string]string{m365SiteID: "todo-name-" + m365SiteID}
nameToID = map[string]string{"todo-name-" + m365SiteID: m365SiteID}
)
// some tests require an existing backup
sel := selectors.NewSharePointBackup(sites)
sel.Include(sel.LibraryFolders(selectors.Any()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector, idToName, nameToID)
require.NoError(t, err, clues.ToCore(err))
err = suite.backupOp.Run(ctx)

View File

@ -11,7 +11,6 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -108,13 +107,17 @@ func (suite *SharePointSuite) TestValidateSharePointBackupCreateFlags() {
}
func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
comboString := []string{"id_1", "id_2"}
gc := &connector.GraphConnector{
Sites: map[string]string{
"url_1": "id_1",
"url_2": "id_2",
},
}
const (
id1 = "id_1"
id2 = "id_2"
url1 = "url_1/foo"
url2 = "url_2/bar"
)
var (
bothIDs = []string{id1, id2}
urlToID = map[string]string{url1: id1, url2: id2}
)
table := []struct {
name string
@ -137,73 +140,72 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
{
name: "site wildcard",
site: []string{utils.Wildcard},
expect: selectors.Any(),
expect: bothIDs,
expectScopesLen: 2,
},
{
name: "url wildcard",
weburl: []string{utils.Wildcard},
expect: selectors.Any(),
expect: bothIDs,
expectScopesLen: 2,
},
{
name: "sites",
site: []string{"id_1", "id_2"},
expect: []string{"id_1", "id_2"},
site: []string{id1, id2},
expect: []string{id1, id2},
expectScopesLen: 2,
},
{
name: "urls",
weburl: []string{"url_1", "url_2"},
expect: []string{"id_1", "id_2"},
weburl: []string{url1, url2},
expect: []string{url1, url2},
expectScopesLen: 2,
},
{
name: "mix sites and urls",
site: []string{"id_1"},
weburl: []string{"url_2"},
expect: []string{"id_1", "id_2"},
site: []string{id1},
weburl: []string{url2},
expect: []string{id1, url2},
expectScopesLen: 2,
},
{
name: "duplicate sites and urls",
site: []string{"id_1", "id_2"},
weburl: []string{"url_1", "url_2"},
expect: comboString,
site: []string{id1, id2},
weburl: []string{url1, url2},
expect: []string{id1, id2, url1, url2},
expectScopesLen: 2,
},
{
name: "unnecessary site wildcard",
site: []string{"id_1", utils.Wildcard},
weburl: []string{"url_1", "url_2"},
expect: selectors.Any(),
site: []string{id1, utils.Wildcard},
weburl: []string{url1, url2},
expect: bothIDs,
expectScopesLen: 2,
},
{
name: "unnecessary url wildcard",
site: comboString,
weburl: []string{"url_1", utils.Wildcard},
expect: selectors.Any(),
site: []string{id1},
weburl: []string{url1, utils.Wildcard},
expect: bothIDs,
expectScopesLen: 2,
},
{
name: "Pages",
site: comboString,
site: bothIDs,
data: []string{dataPages},
expect: comboString,
expect: bothIDs,
expectScopesLen: 1,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext()
defer flush()
sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, test.data, gc)
require.NoError(t, err, clues.ToCore(err))
t := suite.T()
sel, err := sharePointBackupCreateSelectors(ctx, urlToID, test.site, test.weburl, test.data)
require.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners())
})
}

View File

@ -73,7 +73,12 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() {
suite.vpr, suite.cfgFP = tester.MakeTempTestConfigClone(t, force)
suite.m365UserID = tester.M365UserID(t)
users := []string{suite.m365UserID}
var (
users = []string{suite.m365UserID}
idToName = map[string]string{suite.m365UserID: "todo-name-" + suite.m365UserID}
nameToID = map[string]string{"todo-name-" + suite.m365UserID: suite.m365UserID}
)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
@ -100,7 +105,7 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() {
sel.Include(scopes)
bop, err := suite.repo.NewBackup(ctx, sel.Selector)
bop, err := suite.repo.NewBackup(ctx, sel.Selector, idToName, nameToID)
require.NoError(t, err, clues.ToCore(err))
err = bop.Run(ctx)

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/clues"
"github.com/google/uuid"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common"
@ -96,7 +97,7 @@ func generateAndRestoreItems(
print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
return gc.RestoreDataCollections(ctx, version.Backup, acct, sel, dest, opts, dataColls, errs)
return gc.ConsumeRestoreCollections(ctx, version.Backup, acct, sel, dest, opts, dataColls, errs)
}
// ------------------------------------------------------------------------------------------
@ -126,12 +127,12 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
errs := fault.New(false)
normUsers := map[string]struct{}{}
users, err := m365.UserPNs(ctx, acct, errs)
idToName, _, err := m365.UsersMap(ctx, acct, errs)
if err != nil {
return nil, account.Account{}, clues.Wrap(err, "getting tenant users")
}
for _, k := range users {
for _, k := range maps.Keys(idToName) {
normUsers[strings.ToLower(k)] = struct{}{}
}

View File

@ -0,0 +1,6 @@
package common
type IDNamer interface {
ID() string
Name() string
}

View File

@ -43,3 +43,10 @@ func OrNow(t *time.Time) time.Time {
return *t
}
// To generates a pointer from any value. Primarily useful
// for generating pointers to strings and other primitives
// without needing to store a second variable.
func To[T any](t T) *T {
return &t
}

View File

@ -5,7 +5,9 @@ import (
"strings"
"github.com/alcionai/clues"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/discovery"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/exchange"
@ -27,29 +29,33 @@ import (
// Data Collections
// ---------------------------------------------------------------------------
// DataCollections utility function to launch backup operations for exchange and
// onedrive. metadataCols contains any collections with metadata files that may
// be useful for the current backup. Metadata can include things like delta
// tokens or the previous backup's folder hierarchy. The absence of metadataCols
// results in all data being pulled.
func (gc *GraphConnector) DataCollections(
// ProduceBackupCollections generates a slice of backup collections for the service
// specified in the selectors.
// The metadata field can include things like delta tokens or the previous backup's
// folder hierarchy. The absence of metadata causes the collection creation to ignore
// prior history (ie, incrementals) and run a full backup.
func (gc *GraphConnector) ProduceBackupCollections(
ctx context.Context,
owner common.IDNamer,
sels selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]map[string]struct{}, error) {
ctx, end := diagnostics.Span(ctx, "gc:dataCollections", diagnostics.Index("service", sels.Service.String()))
ctx, end := diagnostics.Span(
ctx,
"gc:produceBackupCollections",
diagnostics.Index("service", sels.Service.String()))
defer end()
err := verifyBackupInputs(sels, gc.GetSiteIDs())
err := verifyBackupInputs(sels, maps.Keys(gc.ResourceOwnerIDToName))
if err != nil {
return nil, nil, clues.Stack(err).WithClues(ctx)
}
serviceEnabled, err := checkServiceEnabled(
ctx,
gc.Owners.Users(),
gc.Discovery.Users(),
path.ServiceType(sels.Service),
sels.DiscreteOwner)
if err != nil {
@ -188,10 +194,10 @@ func checkServiceEnabled(
return true, nil
}
// RestoreDataCollections restores data from the specified collections
// ConsumeRestoreCollections restores data from the specified collections
// into M365 using the GraphAPI.
// SideEffect: gc.status is updated at the completion of operation
func (gc *GraphConnector) RestoreDataCollections(
func (gc *GraphConnector) ConsumeRestoreCollections(
ctx context.Context,
backupVersion int,
acct account.Account,

View File

@ -129,8 +129,8 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
}
}
status := connector.AwaitStatus()
assert.NotZero(t, status.Metrics.Successes)
status := connector.Wait()
assert.NotZero(t, status.Successes)
t.Log(status.String())
})
}
@ -205,9 +205,10 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestDataCollections_invali
suite.Run(test.name, func() {
t := suite.T()
collections, excludes, err := connector.DataCollections(
collections, excludes, err := connector.ProduceBackupCollections(
ctx,
test.getSelector(t),
test.getSelector(t),
nil,
control.Options{},
fault.New(true))
@ -286,8 +287,8 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
}
}
status := connector.AwaitStatus()
assert.NotZero(t, status.Metrics.Successes)
status := connector.Wait()
assert.NotZero(t, status.Successes)
t.Log(status.String())
})
}
@ -336,9 +337,10 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
sel := selectors.NewSharePointBackup(siteIDs)
sel.Include(sel.LibraryFolders([]string{"foo"}, selectors.PrefixMatch()))
cols, excludes, err := gc.DataCollections(
cols, excludes, err := gc.ProduceBackupCollections(
ctx,
sel.Selector,
sel.Selector,
nil,
control.Options{},
fault.New(true))
@ -374,9 +376,10 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
sel := selectors.NewSharePointBackup(siteIDs)
sel.Include(sel.Lists(selectors.Any(), selectors.PrefixMatch()))
cols, excludes, err := gc.DataCollections(
cols, excludes, err := gc.ProduceBackupCollections(
ctx,
sel.Selector,
sel.Selector,
nil,
control.Options{},
fault.New(true))

View File

@ -0,0 +1,198 @@
package api
import (
"context"
"fmt"
"regexp"
"strings"
"github.com/alcionai/clues"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/betasdk/sites"
"github.com/alcionai/corso/src/pkg/fault"
)
// ---------------------------------------------------------------------------
// controller
// ---------------------------------------------------------------------------
func (c Client) Sites() Sites {
return Sites{c}
}
// Sites is an interface-compliant provider of the client.
type Sites struct {
Client
}
// ---------------------------------------------------------------------------
// methods
// ---------------------------------------------------------------------------
// GetAll retrieves all sites.
func (c Sites) GetAll(ctx context.Context, errs *fault.Bus) ([]models.Siteable, error) {
service, err := c.service()
if err != nil {
return nil, err
}
var resp models.SiteCollectionResponseable
resp, err = service.Client().Sites().Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting all sites")
}
iter, err := msgraphgocore.NewPageIterator(
resp,
service.Adapter(),
models.CreateSiteCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, graph.Wrap(ctx, err, "creating sites iterator")
}
var (
us = make([]models.Siteable, 0)
el = errs.Local()
)
iterator := func(item any) bool {
if el.Failure() != nil {
return false
}
s, err := validateSite(item)
if errors.Is(err, errKnownSkippableCase) {
// safe to no-op
return true
}
if err != nil {
el.AddRecoverable(graph.Wrap(ctx, err, "validating site"))
return true
}
us = append(us, s)
return true
}
if err := iter.Iterate(ctx, iterator); err != nil {
return nil, graph.Wrap(ctx, err, "iterating all sites")
}
return us, el.Failure()
}
const uuidRE = "[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}"
// matches a site ID, with or without a doman name. Ex, either one of:
// 10rqc2.sharepoint.com,deadbeef-0000-0000-0000-000000000000,beefdead-0000-0000-0000-000000000000
// deadbeef-0000-0000-0000-000000000000,beefdead-0000-0000-0000-000000000000
var siteIDRE = regexp.MustCompile("(.+,)?" + uuidRE + "," + uuidRE)
const webURLGetTemplate = "https://graph.microsoft.com/v1.0/sites/%s:/%s"
// GetByID looks up the site matching the given ID. The ID can be either a
// canonical site id or a webURL. Assumes the webURL is complete and well formed;
// eg: https://10rqc2.sharepoint.com/sites/Example
func (c Sites) GetByID(ctx context.Context, id string) (models.Siteable, error) {
var (
resp models.Siteable
err error
)
ctx = clues.Add(ctx, "given_site_id", id)
if siteIDRE.MatchString(id) {
resp, err = c.stable.Client().SitesById(id).Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting site by id")
}
} else {
var (
url = strings.TrimPrefix(id, "https://")
parts = strings.SplitN(url, "/", 1)
host = parts[0]
path string
)
if len(parts) > 1 {
path = parts[1]
}
rawURL := fmt.Sprintf(webURLGetTemplate, host, path)
resp, err = sites.
NewItemSitesSiteItemRequestBuilder(rawURL, c.stable.Adapter()).
Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting site by weburl")
}
}
return resp, err
}
// GetIDAndName looks up the site matching the given ID, and returns
// its canonical ID and the webURL as the name. Accepts an ID or a
// WebURL as an ID.
func (c Sites) GetIDAndName(ctx context.Context, siteID string) (string, string, error) {
s, err := c.GetByID(ctx, siteID)
if err != nil {
return "", "", err
}
return ptr.Val(s.GetId()), ptr.Val(s.GetWebUrl()), nil
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
var errKnownSkippableCase = clues.New("case is known and skippable")
const personalSitePath = "sharepoint.com/personal/"
// validateSite ensures the item is a Siteable, and contains the necessary
// identifiers that we handle with all users.
// returns the item as a Siteable model.
func validateSite(item any) (models.Siteable, error) {
m, ok := item.(models.Siteable)
if !ok {
return nil, clues.New(fmt.Sprintf("unexpected model: %T", item))
}
id, ok := ptr.ValOK(m.GetId())
if !ok || len(id) == 0 {
return nil, clues.New("missing ID")
}
url, ok := ptr.ValOK(m.GetWebUrl())
if !ok || len(url) == 0 {
return nil, clues.New("missing webURL").With("site_id", id) // TODO: pii
}
// personal (ie: oneDrive) sites have to be filtered out server-side.
if ok && strings.Contains(url, personalSitePath) {
return nil, clues.Stack(errKnownSkippableCase).
With("site_id", id, "site_url", url) // TODO: pii
}
if name, ok := ptr.ValOK(m.GetDisplayName()); !ok || len(name) == 0 {
// the built-in site at "https://{tenant-domain}/search" never has a name.
if strings.HasSuffix(url, "/search") {
return nil, clues.Stack(errKnownSkippableCase).
With("site_id", id, "site_url", url) // TODO: pii
}
return nil, clues.New("missing site display name").With("site_id", id)
}
return m, nil
}

View File

@ -0,0 +1,166 @@
package api
import (
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
)
type SitesUnitSuite struct {
tester.Suite
}
func TestSitesUnitSuite(t *testing.T) {
suite.Run(t, &SitesUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *SitesUnitSuite) TestValidateSite() {
site := models.NewSite()
site.SetWebUrl(ptr.To("sharepoint.com/sites/foo"))
site.SetDisplayName(ptr.To("testsite"))
site.SetId(ptr.To("testID"))
tests := []struct {
name string
args interface{}
want models.Siteable
errCheck assert.ErrorAssertionFunc
errIsSkippable bool
}{
{
name: "Invalid type",
args: string("invalid type"),
errCheck: assert.Error,
},
{
name: "No ID",
args: models.NewSite(),
errCheck: assert.Error,
},
{
name: "No WebURL",
args: func() *models.Site {
s := models.NewSite()
s.SetId(ptr.To("id"))
return s
}(),
errCheck: assert.Error,
},
{
name: "No name",
args: func() *models.Site {
s := models.NewSite()
s.SetId(ptr.To("id"))
s.SetWebUrl(ptr.To("sharepoint.com/sites/foo"))
return s
}(),
errCheck: assert.Error,
},
{
name: "Search site",
args: func() *models.Site {
s := models.NewSite()
s.SetId(ptr.To("id"))
s.SetWebUrl(ptr.To("sharepoint.com/search"))
return s
}(),
errCheck: assert.Error,
errIsSkippable: true,
},
{
name: "Personal OneDrive",
args: func() *models.Site {
s := models.NewSite()
s.SetId(ptr.To("id"))
s.SetWebUrl(ptr.To("https://" + personalSitePath + "/someone's/onedrive"))
return s
}(),
errCheck: assert.Error,
errIsSkippable: true,
},
{
name: "Valid Site",
args: site,
want: site,
errCheck: assert.NoError,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
got, err := validateSite(test.args)
test.errCheck(t, err, clues.ToCore(err))
if test.errIsSkippable {
assert.ErrorIs(t, err, errKnownSkippableCase)
}
assert.Equal(t, test.want, got)
})
}
}
type SitesIntgSuite struct {
tester.Suite
}
func TestSitesIntgSuite(t *testing.T) {
suite.Run(t, &SitesIntgSuite{
Suite: tester.NewIntegrationSuite(t, [][]string{tester.M365AcctCredEnvs}),
})
}
func (suite *SitesIntgSuite) TestSites_GetByID() {
var (
t = suite.T()
siteID = tester.M365SiteID(t)
host = strings.Split(siteID, ",")[0]
shortID = strings.TrimPrefix(siteID, host+",")
siteURL = tester.M365SiteURL(t)
acct = tester.NewM365Account(t)
)
creds, err := acct.M365Config()
require.NoError(t, err, clues.ToCore(err))
client, err := NewClient(creds)
require.NoError(t, err, clues.ToCore(err))
sitesAPI := client.Sites()
table := []struct {
name string
id string
expectErr assert.ErrorAssertionFunc
}{
{"3 part id", siteID, assert.NoError},
{"2 part id", shortID, assert.NoError},
{"malformed id", uuid.NewString(), assert.Error},
{"random id", uuid.NewString() + "," + uuid.NewString(), assert.Error},
{"url", siteURL, assert.NoError},
{"host only", host, assert.NoError},
{"malformed url", "barunihlda", assert.Error},
{"non-matching url", "https://test/sites/testing", assert.Error},
}
for _, test := range table {
suite.Run(test.name, func() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
_, err := sitesAPI.GetByID(ctx, test.id)
test.expectErr(t, err, clues.ToCore(err))
})
}
}

View File

@ -10,6 +10,7 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -150,6 +151,17 @@ func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, err
return resp, err
}
// GetIDAndName looks up the user matching the given ID, and returns
// its canonical ID and the PrincipalName as the name.
func (c Users) GetIDAndName(ctx context.Context, userID string) (string, string, error) {
u, err := c.GetByID(ctx, userID)
if err != nil {
return "", "", err
}
return ptr.Val(u.GetId()), ptr.Val(u.GetUserPrincipalName()), nil
}
func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) {
// Assume all services are enabled
// then filter down to only services the user has enabled

View File

@ -29,6 +29,24 @@ type getWithInfoer interface {
getInfoer
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
func apiClient(ctx context.Context, acct account.Account) (api.Client, error) {
m365, err := acct.M365Config()
if err != nil {
return api.Client{}, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx)
}
client, err := api.NewClient(m365)
if err != nil {
return api.Client{}, clues.Wrap(err, "creating api client").WithClues(ctx)
}
return client, nil
}
// ---------------------------------------------------------------------------
// api
// ---------------------------------------------------------------------------
@ -39,19 +57,15 @@ func Users(
acct account.Account,
errs *fault.Bus,
) ([]models.Userable, error) {
m365, err := acct.M365Config()
client, err := apiClient(ctx, acct)
if err != nil {
return nil, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx)
}
client, err := api.NewClient(m365)
if err != nil {
return nil, clues.Wrap(err, "creating api client").WithClues(ctx)
return nil, err
}
return client.Users().GetAll(ctx, errs)
}
// User fetches a single user's data.
func User(ctx context.Context, gwi getWithInfoer, userID string) (models.Userable, *api.UserInfo, error) {
u, err := gwi.GetByID(ctx, userID)
if err != nil {
@ -69,3 +83,17 @@ func User(ctx context.Context, gwi getWithInfoer, userID string) (models.Userabl
return u, ui, nil
}
// Sites fetches all sharepoint sites in the tenant
func Sites(
ctx context.Context,
acct account.Account,
errs *fault.Bus,
) ([]models.Siteable, error) {
client, err := apiClient(ctx, acct)
if err != nil {
return nil, err
}
return client.Sites().GetAll(ctx, errs)
}

View File

@ -31,10 +31,11 @@ func (suite *DiscoveryIntegrationSuite) TestUsers() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
acct := tester.NewM365Account(t)
errs := fault.New(true)
var (
t = suite.T()
acct = tester.NewM365Account(t)
errs = fault.New(true)
)
users, err := discovery.Users(ctx, acct, errs)
assert.NoError(t, err, clues.ToCore(err))
@ -42,8 +43,7 @@ func (suite *DiscoveryIntegrationSuite) TestUsers() {
ferrs := errs.Errors()
assert.Nil(t, ferrs.Failure)
assert.Empty(t, ferrs.Recovered)
assert.Less(t, 0, len(users))
assert.NotEmpty(t, users)
}
func (suite *DiscoveryIntegrationSuite) TestUsers_InvalidCredentials() {
@ -84,16 +84,85 @@ func (suite *DiscoveryIntegrationSuite) TestUsers_InvalidCredentials() {
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
var (
t = suite.T()
a = test.acct(t)
errs = fault.New(true)
)
users, err := discovery.Users(ctx, a, errs)
assert.Empty(t, users, "returned some users")
assert.NotNil(t, err)
})
}
}
func (suite *DiscoveryIntegrationSuite) TestSites() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
acct = tester.NewM365Account(t)
errs = fault.New(true)
)
sites, err := discovery.Sites(ctx, acct, errs)
assert.NoError(t, err, clues.ToCore(err))
ferrs := errs.Errors()
assert.Nil(t, ferrs.Failure)
assert.Empty(t, ferrs.Recovered)
assert.NotEmpty(t, sites)
}
func (suite *DiscoveryIntegrationSuite) TestSites_InvalidCredentials() {
ctx, flush := tester.NewContext()
defer flush()
table := []struct {
name string
acct func(t *testing.T) account.Account
}{
{
name: "Invalid Credentials",
acct: func(t *testing.T) account.Account {
a, err := account.NewAccount(
account.ProviderM365,
account.M365Config{
M365: credentials.M365{
AzureClientID: "Test",
AzureClientSecret: "without",
},
AzureTenantID: "data",
},
)
require.NoError(t, err, clues.ToCore(err))
return a
},
},
{
name: "Empty Credentials",
acct: func(t *testing.T) account.Account {
// intentionally swallowing the error here
a, _ := account.NewAccount(account.ProviderM365)
return a
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
var (
t = suite.T()
a = test.acct(t)
errs = fault.New(true)
)
a := test.acct(t)
errs := fault.New(true)
users, err := discovery.Users(ctx, a, errs)
assert.Empty(t, users, "returned some users")
assert.NotNil(t, err)
// TODO(ashmrtn): Uncomment when fault package is used in discovery API.
// assert.NotNil(t, errs.Err())
})
}
}

View File

@ -4,25 +4,18 @@ package connector
import (
"context"
"fmt"
"net/http"
"runtime/trace"
"strings"
"sync"
"github.com/alcionai/clues"
"github.com/microsoft/kiota-abstractions-go/serialization"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
@ -37,13 +30,19 @@ import (
// bookkeeping and interfacing with other component.
type GraphConnector struct {
Service graph.Servicer
Owners api.Client
Discovery api.Client
itemClient *http.Client // configured to handle large item downloads
tenant string
Sites map[string]string // webURL -> siteID and siteID -> webURL
credentials account.M365Config
ownerLookup getOwnerIDAndNamer
// maps of resource owner ids to names, and names to ids.
// not guaranteed to be populated, only here as a post-population
// reference for processes that choose to populate the values.
ResourceOwnerIDToName map[string]string
ResourceOwnerNameToID map[string]string
// wg is used to track completion of GC tasks
wg *sync.WaitGroup
region *trace.Region
@ -53,15 +52,6 @@ type GraphConnector struct {
status support.ConnectorOperationStatus // contains the status of the last run status
}
type resource int
const (
UnknownResource resource = iota
AllResources
Users
Sites
)
func NewGraphConnector(
ctx context.Context,
itemClient *http.Client,
@ -69,43 +59,93 @@ func NewGraphConnector(
r resource,
errs *fault.Bus,
) (*GraphConnector, error) {
m365, err := acct.M365Config()
creds, err := acct.M365Config()
if err != nil {
return nil, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx)
}
gc := GraphConnector{
itemClient: itemClient,
tenant: m365.AzureTenantID,
wg: &sync.WaitGroup{},
credentials: m365,
}
gc.Service, err = gc.createService()
service, err := createService(creds)
if err != nil {
return nil, clues.Wrap(err, "creating service connection").WithClues(ctx)
}
gc.Owners, err = api.NewClient(m365)
discovery, err := api.NewClient(creds)
if err != nil {
return nil, clues.Wrap(err, "creating api client").WithClues(ctx)
}
if r == AllResources || r == Sites {
if err = gc.setTenantSites(ctx, errs); err != nil {
return nil, clues.Wrap(err, "retrieveing tenant site list")
}
rc, err := r.resourceClient(discovery)
if err != nil {
return nil, clues.Wrap(err, "creating resource client").WithClues(ctx)
}
gc := GraphConnector{
itemClient: itemClient,
Discovery: discovery,
tenant: acct.ID(),
wg: &sync.WaitGroup{},
credentials: creds,
ownerLookup: rc,
Service: service,
}
return &gc, nil
}
// ---------------------------------------------------------------------------
// Owner Lookup
// ---------------------------------------------------------------------------
// PopulateOwnerIDAndNamesFrom takes the provided owner identifier and produces
// the owner's name and ID from that value. Returns an error if the owner is
// not recognized by the current tenant.
//
// The id-name maps are optional. Some processes will look up all owners in
// the tenant before reaching this step. In that case, the data gets handed
// down for this func to consume instead of performing further queries. The
// maps get stored inside the gc instance for later re-use.
//
// TODO: If the maps are nil or empty, this func will perform a lookup on the given
// owner, and populate each map with that owner's id and name for downstream
// guarantees about that data being present. Optional performance enhancement
// idea: downstream from here, we should _only_ need the given user's id and name,
// and could store minimal map copies with that info instead of the whole tenant.
func (gc *GraphConnector) PopulateOwnerIDAndNamesFrom(
ctx context.Context,
owner string, // input value, can be either id or name
idToName, nameToID map[string]string, // optionally pre-populated lookups
) (string, string, error) {
// ensure the maps exist, even if they aren't populated so that
// getOwnerIDAndNameFrom can populate any values it looks up.
if len(idToName) == 0 {
idToName = map[string]string{}
}
if len(nameToID) == 0 {
nameToID = map[string]string{}
}
id, name, err := gc.ownerLookup.getOwnerIDAndNameFrom(ctx, gc.Discovery, owner, idToName, nameToID)
if err != nil {
return "", "", errors.Wrap(err, "resolving resource owner details")
}
gc.ResourceOwnerIDToName = idToName
gc.ResourceOwnerNameToID = nameToID
return id, name, nil
}
// ---------------------------------------------------------------------------
// Service Client
// ---------------------------------------------------------------------------
// createService constructor for graphService component
func (gc *GraphConnector) createService() (*graph.Service, error) {
func createService(creds account.M365Config) (*graph.Service, error) {
adapter, err := graph.CreateAdapter(
gc.credentials.AzureTenantID,
gc.credentials.AzureClientID,
gc.credentials.AzureClientSecret)
creds.AzureTenantID,
creds.AzureClientID,
creds.AzureClientSecret)
if err != nil {
return &graph.Service{}, err
}
@ -113,117 +153,12 @@ func (gc *GraphConnector) createService() (*graph.Service, error) {
return graph.NewService(adapter), nil
}
// setTenantSites queries the M365 to identify the sites in the
// workspace. The sites field is updated during this method
// iff the returned error is nil.
func (gc *GraphConnector) setTenantSites(ctx context.Context, errs *fault.Bus) error {
gc.Sites = map[string]string{}
ctx, end := diagnostics.Span(ctx, "gc:setTenantSites")
defer end()
sites, err := getResources(
ctx,
gc.Service,
gc.tenant,
sharepoint.GetAllSitesForTenant,
models.CreateSiteCollectionResponseFromDiscriminatorValue,
identifySite,
errs)
if err != nil {
return err
}
gc.Sites = sites
return nil
}
var errKnownSkippableCase = clues.New("case is known and skippable")
const personalSitePath = "sharepoint.com/personal/"
// Transforms an interface{} into a key,value pair representing
// siteName:siteID.
func identifySite(item any) (string, string, error) {
m, ok := item.(models.Siteable)
if !ok {
return "", "", clues.New("non-Siteable item").With("item_type", fmt.Sprintf("%T", item))
}
id := ptr.Val(m.GetId())
url, ok := ptr.ValOK(m.GetWebUrl())
if m.GetName() == nil {
// the built-in site at "https://{tenant-domain}/search" never has a name.
if ok && strings.HasSuffix(url, "/search") {
// TODO: pii siteID, on this and all following cases
return "", "", clues.Stack(errKnownSkippableCase).With("site_id", id)
}
return "", "", clues.New("site has no name").With("site_id", id)
}
// personal (ie: oneDrive) sites have to be filtered out server-side.
if ok && strings.Contains(url, personalSitePath) {
return "", "", clues.Stack(errKnownSkippableCase).With("site_id", id)
}
return url, id, nil
}
// GetSiteWebURLs returns the WebURLs of sharepoint sites within the tenant.
func (gc *GraphConnector) GetSiteWebURLs() []string {
return maps.Keys(gc.Sites)
}
// GetSiteIds returns the canonical site IDs in the tenant
func (gc *GraphConnector) GetSiteIDs() []string {
return maps.Values(gc.Sites)
}
// UnionSiteIDsAndWebURLs reduces the id and url slices into a single slice of site IDs.
// WebURLs will run as a path-suffix style matcher. Callers may provide partial urls, though
// each element in the url must fully match. Ex: the webURL value "foo" will match "www.ex.com/foo",
// but not match "www.ex.com/foobar".
// The returned IDs are reduced to a set of unique values.
func (gc *GraphConnector) UnionSiteIDsAndWebURLs(
ctx context.Context,
ids, urls []string,
errs *fault.Bus,
) ([]string, error) {
if len(gc.Sites) == 0 {
if err := gc.setTenantSites(ctx, errs); err != nil {
return nil, err
}
}
idm := map[string]struct{}{}
for _, id := range ids {
idm[id] = struct{}{}
}
match := filters.PathSuffix(urls)
for url, id := range gc.Sites {
if !match.Compare(url) {
continue
}
idm[id] = struct{}{}
}
idsl := make([]string, 0, len(idm))
for id := range idm {
idsl = append(idsl, id)
}
return idsl, nil
}
// ---------------------------------------------------------------------------
// Processing Status
// ---------------------------------------------------------------------------
// AwaitStatus waits for all gc tasks to complete and then returns status
func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
func (gc *GraphConnector) Wait() *data.CollectionStats {
defer func() {
if gc.region != nil {
gc.region.End()
@ -233,12 +168,18 @@ func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
gc.wg.Wait()
// clean up and reset statefulness
status := gc.status
dcs := data.CollectionStats{
Folders: gc.status.Folders,
Objects: gc.status.Metrics.Objects,
Successes: gc.status.Metrics.Successes,
Bytes: gc.status.Metrics.Bytes,
Details: gc.status.String(),
}
gc.wg = &sync.WaitGroup{}
gc.status = support.ConnectorOperationStatus{}
return &status
return &dcs
}
// UpdateStatus is used by gc initiated tasks to indicate completion
@ -273,57 +214,110 @@ func (gc *GraphConnector) incrementMessagesBy(num int) {
}
// ---------------------------------------------------------------------------
// Helper Funcs
// Resource Handling
// ---------------------------------------------------------------------------
func getResources(
ctx context.Context,
gs graph.Servicer,
tenantID string,
query func(context.Context, graph.Servicer) (serialization.Parsable, error),
parser func(parseNode serialization.ParseNode) (serialization.Parsable, error),
identify func(any) (string, string, error),
errs *fault.Bus,
) (map[string]string, error) {
resources := map[string]string{}
type resource int
response, err := query(ctx, gs)
if err != nil {
return nil, graph.Wrap(ctx, err, "retrieving tenant's resources")
const (
UnknownResource resource = iota
AllResources // unused
Users
Sites
)
func (r resource) resourceClient(discovery api.Client) (*resourceClient, error) {
switch r {
case Users:
return &resourceClient{enum: r, getter: discovery.Users()}, nil
case Sites:
return &resourceClient{enum: r, getter: discovery.Sites()}, nil
default:
return nil, clues.New("unrecognized owner resource enum").With("resource_enum", r)
}
iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(), parser)
if err != nil {
return nil, graph.Stack(ctx, err)
}
el := errs.Local()
callbackFunc := func(item any) bool {
if el.Failure() != nil {
return false
}
k, v, err := identify(item)
if err != nil {
if !errors.Is(err, errKnownSkippableCase) {
el.AddRecoverable(clues.Stack(err).
WithClues(ctx).
With("query_url", gs.Adapter().GetBaseUrl()))
}
return true
}
resources[k] = v
resources[v] = k
return true
}
if err := iter.Iterate(ctx, callbackFunc); err != nil {
return nil, graph.Stack(ctx, err)
}
return resources, el.Failure()
}
type resourceClient struct {
enum resource
getter getIDAndNamer
}
type getIDAndNamer interface {
GetIDAndName(ctx context.Context, owner string) (
ownerID string,
ownerName string,
err error,
)
}
var _ getOwnerIDAndNamer = &resourceClient{}
type getOwnerIDAndNamer interface {
getOwnerIDAndNameFrom(
ctx context.Context,
discovery api.Client,
owner string,
idToName, nameToID map[string]string,
) (
ownerID string,
ownerName string,
err error,
)
}
var ErrResourceOwnerNotFound = clues.New("resource owner not found in tenant")
// getOwnerIDAndNameFrom looks up the owner's canonical id and display name.
// if idToName and nameToID are populated, and the owner is a key of one of
// those maps, then those values are returned.
//
// As a fallback, the resource calls the discovery api to fetch the user or
// site using the owner value. This fallback assumes that the owner is a well
// formed ID or display name of appropriate design (PrincipalName for users,
// WebURL for sites). If the fallback lookup is used, the maps are populated
// to contain the id and name references.
//
// Consumers are allowed to pass in a path suffix (eg: /sites/foo) as a site
// owner, but only if they also pass in a nameToID map. A nil map will cascade
// to the fallback, which will fail for having a malformed id value.
func (r resourceClient) getOwnerIDAndNameFrom(
ctx context.Context,
discovery api.Client,
owner string,
idToName, nameToID map[string]string,
) (string, string, error) {
if n, ok := idToName[owner]; ok {
return owner, n, nil
} else if id, ok := nameToID[owner]; ok {
return id, owner, nil
}
ctx = clues.Add(ctx, "owner_identifier", owner)
var (
id, name string
err error
)
// check if the provided owner is a suffix of a weburl in the lookup map
if r.enum == Sites {
url, _, ok := filters.PathSuffix([]string{owner}).CompareAny(maps.Keys(nameToID)...)
if ok {
return nameToID[url], url, nil
}
}
id, name, err = r.getter.GetIDAndName(ctx, owner)
if err != nil {
return "", "", err
}
if len(id) == 0 || len(name) == 0 {
return "", "", clues.Stack(ErrResourceOwnerNotFound)
}
idToName[id] = name
nameToID[name] = id
return id, name, nil
}

View File

@ -111,17 +111,16 @@ func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_Status() {
go statusTestTask(&gc, 4, 1, 1)
go statusTestTask(&gc, 4, 1, 1)
status := gc.AwaitStatus()
stats := gc.Wait()
t := suite.T()
assert.NotEmpty(t, gc.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, status.Metrics.Objects)
assert.Equal(t, 8, stats.Objects)
// Expect 2 success
assert.Equal(t, 2, status.Metrics.Successes)
assert.Equal(t, 2, stats.Successes)
// Expect 2 folders
assert.Equal(t, 2, status.Folders)
assert.Equal(t, 2, stats.Folders)
}
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices() {

View File

@ -452,11 +452,11 @@ func (suite *GraphConnectorSharePointIntegrationSuite) SetupSuite() {
si.resourceOwner = tester.M365SiteID(suite.T())
user, err := si.connector.Owners.Users().GetByID(ctx, si.user)
user, err := si.connector.Discovery.Users().GetByID(ctx, si.user)
require.NoError(suite.T(), err, "fetching user", si.user, clues.ToCore(err))
si.userID = ptr.Val(user.GetId())
secondaryUser, err := si.connector.Owners.Users().GetByID(ctx, si.secondaryUser)
secondaryUser, err := si.connector.Discovery.Users().GetByID(ctx, si.secondaryUser)
require.NoError(suite.T(), err, "fetching user", si.secondaryUser, clues.ToCore(err))
si.secondaryUserID = ptr.Val(secondaryUser.GetId())
@ -499,11 +499,11 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) SetupSuite() {
si.resourceOwner = si.user
user, err := si.connector.Owners.Users().GetByID(ctx, si.user)
user, err := si.connector.Discovery.Users().GetByID(ctx, si.user)
require.NoError(suite.T(), err, "fetching user", si.user, clues.ToCore(err))
si.userID = ptr.Val(user.GetId())
secondaryUser, err := si.connector.Owners.Users().GetByID(ctx, si.secondaryUser)
secondaryUser, err := si.connector.Discovery.Users().GetByID(ctx, si.secondaryUser)
require.NoError(suite.T(), err, "fetching user", si.secondaryUser, clues.ToCore(err))
si.secondaryUserID = ptr.Val(secondaryUser.GetId())
@ -695,11 +695,11 @@ func (suite *GraphConnectorOneDriveNightlySuite) SetupSuite() {
si.resourceOwner = si.user
user, err := si.connector.Owners.Users().GetByID(ctx, si.user)
user, err := si.connector.Discovery.Users().GetByID(ctx, si.user)
require.NoError(suite.T(), err, "fetching user", si.user, clues.ToCore(err))
si.userID = ptr.Val(user.GetId())
secondaryUser, err := si.connector.Owners.Users().GetByID(ctx, si.secondaryUser)
secondaryUser, err := si.connector.Discovery.Users().GetByID(ctx, si.secondaryUser)
require.NoError(suite.T(), err, "fetching user", si.secondaryUser, clues.ToCore(err))
si.secondaryUserID = ptr.Val(secondaryUser.GetId())

View File

@ -38,104 +38,175 @@ func TestGraphConnectorUnitSuite(t *testing.T) {
suite.Run(t, &GraphConnectorUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *GraphConnectorUnitSuite) TestUnionSiteIDsAndWebURLs() {
var _ getIDAndNamer = &mockNameIDGetter{}
type mockNameIDGetter struct {
id, name string
}
func (mnig mockNameIDGetter) GetIDAndName(
_ context.Context,
_ string,
) (string, string, error) {
return mnig.id, mnig.name, nil
}
func (suite *GraphConnectorUnitSuite) TestPopulateOwnerIDAndNamesFrom() {
const (
url1 = "www.foo.com/bar"
url2 = "www.fnords.com/smarf"
path1 = "bar"
path2 = "/smarf"
id1 = "site-id-1"
id2 = "site-id-2"
ownerID = "owner-id"
ownerName = "owner-name"
)
gc := &GraphConnector{
// must be populated, else the func will try to make a graph call
// to retrieve site data.
Sites: map[string]string{
url1: id1,
url2: id2,
},
}
var (
itn = map[string]string{ownerID: ownerName}
nti = map[string]string{ownerName: ownerID}
lookup = &resourceClient{
enum: Users,
getter: &mockNameIDGetter{id: ownerID, name: ownerName},
}
noLookup = &resourceClient{enum: Users, getter: &mockNameIDGetter{}}
siteLookup = &resourceClient{enum: Sites, getter: &mockNameIDGetter{}}
)
table := []struct {
name string
ids []string
urls []string
expect []string
name string
owner string
idToName map[string]string
nameToID map[string]string
rc *resourceClient
expectID string
expectName string
expectErr assert.ErrorAssertionFunc
}{
{
name: "nil",
name: "nil maps, getter lookup",
owner: ownerID,
rc: lookup,
idToName: nil,
nameToID: nil,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "empty",
ids: []string{},
urls: []string{},
expect: []string{},
name: "only id map with owner id",
owner: ownerID,
rc: noLookup,
idToName: itn,
nameToID: nil,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "ids only",
ids: []string{id1, id2},
urls: []string{},
expect: []string{id1, id2},
name: "only name map with owner id",
owner: ownerID,
rc: lookup,
idToName: nil,
nameToID: nti,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "urls only",
ids: []string{},
urls: []string{url1, url2},
expect: []string{id1, id2},
name: "only id map with owner name",
owner: ownerName,
rc: lookup,
idToName: itn,
nameToID: nil,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "url suffix only",
ids: []string{},
urls: []string{path1, path2},
expect: []string{id1, id2},
name: "only name map with owner name",
owner: ownerName,
rc: lookup,
idToName: nil,
nameToID: nti,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "url and suffix overlap",
ids: []string{},
urls: []string{url1, url2, path1, path2},
expect: []string{id1, id2},
name: "both maps with owner id",
owner: ownerID,
rc: noLookup,
idToName: itn,
nameToID: nti,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "ids and urls, no overlap",
ids: []string{id1},
urls: []string{url2},
expect: []string{id1, id2},
name: "both maps with owner name",
owner: ownerName,
rc: noLookup,
idToName: itn,
nameToID: nti,
expectID: ownerID,
expectName: ownerName,
expectErr: assert.NoError,
},
{
name: "ids and urls, overlap",
ids: []string{id1, id2},
urls: []string{url1, url2},
expect: []string{id1, id2},
name: "non-matching maps with owner id",
owner: ownerID,
rc: noLookup,
idToName: map[string]string{"foo": "bar"},
nameToID: map[string]string{"fnords": "smarf"},
expectID: "",
expectName: "",
expectErr: assert.Error,
},
{
name: "partial non-match on path",
ids: []string{},
urls: []string{path1[2:], path2[2:]},
expect: []string{},
name: "non-matching with owner name",
owner: ownerName,
rc: noLookup,
idToName: map[string]string{"foo": "bar"},
nameToID: map[string]string{"fnords": "smarf"},
expectID: "",
expectName: "",
expectErr: assert.Error,
},
{
name: "partial non-match on url",
ids: []string{},
urls: []string{url1[5:], url2[5:]},
expect: []string{},
name: "site suffix lookup",
owner: "/url/path",
rc: siteLookup,
idToName: nil,
nameToID: map[string]string{"http://some/site/url/path": ownerID},
expectID: ownerID,
expectName: "http://some/site/url/path",
expectErr: assert.NoError,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext()
defer flush()
result, err := gc.UnionSiteIDsAndWebURLs(ctx, test.ids, test.urls, fault.New(true))
assert.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, test.expect, result)
var (
t = suite.T()
gc = &GraphConnector{ownerLookup: test.rc}
)
id, name, err := gc.PopulateOwnerIDAndNamesFrom(
ctx,
test.owner,
test.idToName,
test.nameToID)
test.expectErr(t, err, clues.ToCore(err))
if err != nil {
return
}
assert.Equal(t, test.expectID, id, "id")
assert.Equal(t, test.expectName, name, "name")
})
}
}
func (suite *GraphConnectorUnitSuite) TestGraphConnector_AwaitStatus() {
func (suite *GraphConnectorUnitSuite) TestGraphConnector_Wait() {
ctx, flush := tester.NewContext()
defer flush()
@ -156,14 +227,14 @@ func (suite *GraphConnectorUnitSuite) TestGraphConnector_AwaitStatus() {
gc.wg.Add(1)
gc.UpdateStatus(status)
result := gc.AwaitStatus()
result := gc.Wait()
require.NotNil(t, result)
assert.Nil(t, gc.region, "region")
assert.Empty(t, gc.status, "status")
assert.Equal(t, 1, result.Folders)
assert.Equal(t, 2, result.Metrics.Objects)
assert.Equal(t, 3, result.Metrics.Successes)
assert.Equal(t, int64(4), result.Metrics.Bytes)
assert.Equal(t, 2, result.Objects)
assert.Equal(t, 3, result.Successes)
assert.Equal(t, int64(4), result.Bytes)
}
// ---------------------------------------------------------------------------
@ -199,35 +270,6 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() {
tester.LogTimeOfTest(suite.T())
}
// TestSetTenantSites verifies GraphConnector's ability to query
// the sites associated with the credentials
func (suite *GraphConnectorIntegrationSuite) TestSetTenantSites() {
newConnector := GraphConnector{
tenant: "test_tenant",
Sites: make(map[string]string, 0),
credentials: suite.connector.credentials,
}
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
service, err := newConnector.createService()
require.NoError(t, err, clues.ToCore(err))
newConnector.Service = service
assert.Equal(t, 0, len(newConnector.Sites))
err = newConnector.setTenantSites(ctx, fault.New(true))
assert.NoError(t, err, clues.ToCore(err))
assert.Less(t, 0, len(newConnector.Sites))
for _, site := range newConnector.Sites {
assert.NotContains(t, "sharepoint.com/personal/", site)
}
}
func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
ctx, flush := tester.NewContext()
defer flush()
@ -241,7 +283,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
}
)
deets, err := suite.connector.RestoreDataCollections(
deets, err := suite.connector.ConsumeRestoreCollections(
ctx,
version.Backup,
acct,
@ -256,10 +298,10 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
assert.Error(t, err, clues.ToCore(err))
assert.NotNil(t, deets)
status := suite.connector.AwaitStatus()
assert.Equal(t, 0, status.Metrics.Objects)
status := suite.connector.Wait()
assert.Equal(t, 0, status.Objects)
assert.Equal(t, 0, status.Folders)
assert.Equal(t, 0, status.Metrics.Successes)
assert.Equal(t, 0, status.Successes)
}
func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
@ -320,7 +362,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
ctx, flush := tester.NewContext()
defer flush()
deets, err := suite.connector.RestoreDataCollections(
deets, err := suite.connector.ConsumeRestoreCollections(
ctx,
version.Backup,
suite.acct,
@ -335,10 +377,10 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
require.NoError(t, err, clues.ToCore(err))
assert.NotNil(t, deets)
stats := suite.connector.AwaitStatus()
assert.Zero(t, stats.Metrics.Objects)
stats := suite.connector.Wait()
assert.Zero(t, stats.Objects)
assert.Zero(t, stats.Folders)
assert.Zero(t, stats.Metrics.Successes)
assert.Zero(t, stats.Successes)
})
}
}
@ -400,7 +442,7 @@ func runRestore(
restoreGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), config.resource)
restoreSel := getSelectorWith(t, config.service, config.resourceOwners, true)
deets, err := restoreGC.RestoreDataCollections(
deets, err := restoreGC.ConsumeRestoreCollections(
ctx,
backupVersion,
config.acct,
@ -412,11 +454,11 @@ func runRestore(
require.NoError(t, err, clues.ToCore(err))
assert.NotNil(t, deets)
status := restoreGC.AwaitStatus()
status := restoreGC.Wait()
runTime := time.Since(start)
assert.Equal(t, numRestoreItems, status.Metrics.Objects, "restored status.Metrics.Objects")
assert.Equal(t, numRestoreItems, status.Metrics.Successes, "restored status.Metrics.Successes")
assert.Equal(t, numRestoreItems, status.Objects, "restored status.Objects")
assert.Equal(t, numRestoreItems, status.Successes, "restored status.Successes")
assert.Len(
t,
deets.Entries,
@ -457,9 +499,10 @@ func runBackupAndCompare(
t.Logf("Selective backup of %s\n", backupSel)
start := time.Now()
dcs, excludes, err := backupGC.DataCollections(
dcs, excludes, err := backupGC.ProduceBackupCollections(
ctx,
backupSel,
backupSel,
nil,
config.opts,
fault.New(true))
@ -480,12 +523,12 @@ func runBackupAndCompare(
config.dest,
config.opts.RestorePermissions)
status := backupGC.AwaitStatus()
status := backupGC.Wait()
assert.Equalf(t, totalItems+skipped, status.Metrics.Objects,
"backup status.Metrics.Objects; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Metrics.Successes,
"backup status.Metrics.Successes; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Objects,
"backup status.Objects; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Successes,
"backup status.Successes; wanted %d items + %d skipped", totalItems, skipped)
}
func runRestoreBackupTest(
@ -964,7 +1007,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
)
restoreGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource)
deets, err := restoreGC.RestoreDataCollections(
deets, err := restoreGC.ConsumeRestoreCollections(
ctx,
version.Backup,
suite.acct,
@ -979,12 +1022,12 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
require.NoError(t, err, clues.ToCore(err))
require.NotNil(t, deets)
status := restoreGC.AwaitStatus()
status := restoreGC.Wait()
// Always just 1 because it's just 1 collection.
assert.Equal(t, totalItems, status.Metrics.Objects, "status.Metrics.Objects")
assert.Equal(t, totalItems, status.Metrics.Successes, "status.Metrics.Successes")
assert.Len(
t, deets.Entries, totalItems,
assert.Equal(t, totalItems, status.Objects, "status.Objects")
assert.Equal(t, totalItems, status.Successes, "status.Successes")
assert.Equal(
t, totalItems, len(deets.Entries),
"details entries contains same item count as total successful items restored")
t.Log("Restore complete")
@ -996,9 +1039,10 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
backupSel := backupSelectorForExpected(t, test.service, expectedDests)
t.Log("Selective backup of", backupSel)
dcs, excludes, err := backupGC.DataCollections(
dcs, excludes, err := backupGC.ProduceBackupCollections(
ctx,
backupSel,
backupSel,
nil,
control.Options{
RestorePermissions: true,
@ -1023,9 +1067,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
control.RestoreDestination{},
true)
status := backupGC.AwaitStatus()
assert.Equal(t, allItems+skipped, status.Metrics.Objects, "status.Metrics.Objects")
assert.Equal(t, allItems+skipped, status.Metrics.Successes, "status.Metrics.Successes")
status := backupGC.Wait()
assert.Equal(t, allItems+skipped, status.Objects, "status.Objects")
assert.Equal(t, allItems+skipped, status.Successes, "status.Successes")
})
}
}
@ -1147,9 +1191,10 @@ func (suite *GraphConnectorIntegrationSuite) TestBackup_CreatesPrefixCollections
start = time.Now()
)
dcs, excludes, err := backupGC.DataCollections(
dcs, excludes, err := backupGC.ProduceBackupCollections(
ctx,
backupSel,
backupSel,
nil,
control.Options{
RestorePermissions: false,
@ -1191,7 +1236,7 @@ func (suite *GraphConnectorIntegrationSuite) TestBackup_CreatesPrefixCollections
assert.ElementsMatch(t, test.categories, foundCategories)
backupGC.AwaitStatus()
backupGC.Wait()
assert.NoError(t, errs.Failure())
})

View File

@ -0,0 +1,56 @@
package mockconnector
import (
"context"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
)
type GraphConnector struct {
Collections []data.BackupCollection
Exclude map[string]map[string]struct{}
Deets *details.Details
Err error
Stats data.CollectionStats
}
func (gc GraphConnector) ProduceBackupCollections(
_ context.Context,
_ common.IDNamer,
_ selectors.Selector,
_ []data.RestoreCollection,
_ control.Options,
_ *fault.Bus,
) (
[]data.BackupCollection,
map[string]map[string]struct{},
error,
) {
return gc.Collections, gc.Exclude, gc.Err
}
func (gc GraphConnector) Wait() *data.CollectionStats {
return &gc.Stats
}
func (gc GraphConnector) ConsumeRestoreCollections(
_ context.Context,
_ int,
_ account.Account,
_ selectors.Selector,
_ control.RestoreDestination,
_ control.Options,
_ []data.RestoreCollection,
_ *fault.Bus,
) (*details.Details, error) {
return gc.Deets, gc.Err
}

View File

@ -0,0 +1,16 @@
package data
type CollectionStats struct {
Folders int
Objects, Successes int
Bytes int64
Details string
}
func (cs CollectionStats) IsZero() bool {
return cs.Folders+cs.Objects+cs.Successes+int(cs.Bytes) == 0
}
func (cs CollectionStats) String() string {
return cs.Details
}

View File

@ -124,13 +124,13 @@ type PrevRefs struct {
Location path.Path
}
// BackupCollections takes a set of collections and creates a kopia snapshot
// ConsumeBackupCollections takes a set of collections and creates a kopia snapshot
// with the data that they contain. previousSnapshots is used for incremental
// backups and should represent the base snapshot from which metadata is sourced
// from as well as any incomplete snapshot checkpoints that may contain more
// recent data than the base snapshot. The absence of previousSnapshots causes a
// complete backup of all data.
func (w Wrapper) BackupCollections(
func (w Wrapper) ConsumeBackupCollections(
ctx context.Context,
previousSnapshots []IncrementalBase,
collections []data.BackupCollection,
@ -143,7 +143,7 @@ func (w Wrapper) BackupCollections(
return nil, nil, nil, clues.Stack(errNotConnected).WithClues(ctx)
}
ctx, end := diagnostics.Span(ctx, "kopia:backupCollections")
ctx, end := diagnostics.Span(ctx, "kopia:consumeBackupCollections")
defer end()
if len(collections) == 0 && len(globalExcludeSet) == 0 {

View File

@ -276,7 +276,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
suite.Run(test.name, func() {
t := suite.T()
stats, deets, _, err := suite.w.BackupCollections(
stats, deets, _, err := suite.w.ConsumeBackupCollections(
suite.ctx,
prevSnaps,
collections,
@ -423,7 +423,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() {
t := suite.T()
collections := test.cols()
stats, deets, prevShortRefs, err := suite.w.BackupCollections(
stats, deets, prevShortRefs, err := suite.w.ConsumeBackupCollections(
suite.ctx,
prevSnaps,
collections,
@ -525,7 +525,7 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
fp2, err := suite.storePath2.Append(dc2.Names[0], true)
require.NoError(t, err, clues.ToCore(err))
stats, _, _, err := w.BackupCollections(
stats, _, _, err := w.ConsumeBackupCollections(
ctx,
nil,
[]data.BackupCollection{dc1, dc2},
@ -644,7 +644,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
},
}
stats, deets, _, err := suite.w.BackupCollections(
stats, deets, _, err := suite.w.ConsumeBackupCollections(
suite.ctx,
nil,
collections,
@ -706,7 +706,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollectionsHandlesNoCollections()
ctx, flush := tester.NewContext()
defer flush()
s, d, _, err := suite.w.BackupCollections(
s, d, _, err := suite.w.ConsumeBackupCollections(
ctx,
nil,
test.collections,
@ -866,7 +866,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
tags[k] = ""
}
stats, deets, _, err := suite.w.BackupCollections(
stats, deets, _, err := suite.w.ConsumeBackupCollections(
suite.ctx,
nil,
collections,
@ -1018,7 +1018,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() {
}
}
stats, _, _, err := suite.w.BackupCollections(
stats, _, _, err := suite.w.ConsumeBackupCollections(
suite.ctx,
[]IncrementalBase{
{

View File

@ -9,8 +9,6 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/crash"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/events"
@ -34,14 +32,14 @@ import (
type BackupOperation struct {
operation
ResourceOwner string `json:"resourceOwner"`
ResourceOwnerName string `json:"resourceOwnerName"`
ResourceOwner common.IDNamer
Results BackupResults `json:"results"`
Selectors selectors.Selector `json:"selectors"`
Version string `json:"version"`
account account.Account
bp BackupProducer
// when true, this allows for incremental backups instead of full data pulls
incremental bool
@ -60,24 +58,19 @@ func NewBackupOperation(
opts control.Options,
kw *kopia.Wrapper,
sw *store.Wrapper,
gc *connector.GraphConnector,
bp BackupProducer,
acct account.Account,
selector selectors.Selector,
ownerName string,
owner common.IDNamer,
bus events.Eventer,
) (BackupOperation, error) {
op := BackupOperation{
operation: newOperation(opts, bus, kw, sw, gc),
ResourceOwner: selector.DiscreteOwner,
ResourceOwnerName: ownerName,
Selectors: selector,
Version: "v0",
account: acct,
incremental: useIncrementalBackup(selector, opts),
}
if len(ownerName) == 0 {
op.ResourceOwnerName = op.ResourceOwner
operation: newOperation(opts, bus, kw, sw),
ResourceOwner: owner,
Selectors: selector,
Version: "v0",
account: acct,
incremental: useIncrementalBackup(selector, opts),
}
if err := op.validate(); err != nil {
@ -88,10 +81,18 @@ func NewBackupOperation(
}
func (op BackupOperation) validate() error {
if len(op.ResourceOwner) == 0 {
if op.ResourceOwner == nil {
return clues.New("backup requires a resource owner")
}
if len(op.ResourceOwner.ID()) == 0 {
return clues.New("backup requires a resource owner with a populated ID")
}
if op.bp == nil {
return clues.New("missing backup producer")
}
return op.operation.validate()
}
@ -101,7 +102,7 @@ func (op BackupOperation) validate() error {
// get populated asynchronously.
type backupStats struct {
k *kopia.BackupStats
gc *support.ConnectorOperationStatus
gc *data.CollectionStats
resourceCount int
}
@ -160,7 +161,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
// Execution
// -----
observe.Message(ctx, observe.Safe("Backing Up"), observe.Bullet, observe.PII(op.ResourceOwner))
observe.Message(ctx, observe.Safe("Backing Up"), observe.Bullet, observe.PII(op.ResourceOwner.Name()))
deets, err := op.do(
ctx,
@ -243,14 +244,21 @@ func (op *BackupOperation) do(
return nil, clues.Wrap(err, "producing manifests and metadata")
}
cs, excludes, err := produceBackupDataCollections(ctx, op.gc, op.Selectors, mdColls, op.Options, op.Errors)
cs, excludes, err := produceBackupDataCollections(
ctx,
op.bp,
op.ResourceOwner,
op.Selectors,
mdColls,
op.Options,
op.Errors)
if err != nil {
return nil, clues.Wrap(err, "producing backup data collections")
}
ctx = clues.Add(ctx, "coll_count", len(cs))
writeStats, deets, toMerge, err := consumeBackupDataCollections(
writeStats, deets, toMerge, err := consumeBackupCollections(
ctx,
op.kopia,
op.account.ID(),
@ -279,9 +287,9 @@ func (op *BackupOperation) do(
return nil, clues.Wrap(err, "merging details")
}
opStats.gc = op.gc.AwaitStatus()
opStats.gc = op.bp.Wait()
logger.Ctx(ctx).Debug(op.gc.PrintableStatus())
logger.Ctx(ctx).Debug(opStats.gc)
return deets, nil
}
@ -309,10 +317,25 @@ func useIncrementalBackup(sel selectors.Selector, opts control.Options) bool {
// Producer funcs
// ---------------------------------------------------------------------------
type BackupProducer interface {
ProduceBackupCollections(
ctx context.Context,
resourceOwner common.IDNamer,
sels selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]map[string]struct{}, error)
// TODO: ConnectorOperationStatus should be replaced with something
// more generic.
Wait() *data.CollectionStats
}
// calls the producer to generate collections of data to backup
func produceBackupDataCollections(
ctx context.Context,
gc *connector.GraphConnector,
bp BackupProducer,
resourceOwner common.IDNamer,
sel selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
@ -325,15 +348,15 @@ func produceBackupDataCollections(
closer()
}()
return gc.DataCollections(ctx, sel, metadata, ctrlOpts, errs)
return bp.ProduceBackupCollections(ctx, resourceOwner, sel, metadata, ctrlOpts, errs)
}
// ---------------------------------------------------------------------------
// Consumer funcs
// ---------------------------------------------------------------------------
type backuper interface {
BackupCollections(
type BackupConsumer interface {
ConsumeBackupCollections(
ctx context.Context,
bases []kopia.IncrementalBase,
cs []data.BackupCollection,
@ -389,9 +412,9 @@ func builderFromReason(ctx context.Context, tenant string, r kopia.Reason) (*pat
}
// calls kopia to backup the collections of data
func consumeBackupDataCollections(
func consumeBackupCollections(
ctx context.Context,
bu backuper,
bc BackupConsumer,
tenantID string,
reasons []kopia.Reason,
mans []*kopia.ManifestEntry,
@ -465,7 +488,7 @@ func consumeBackupDataCollections(
"base_backup_id", mbID)
}
kopiaStats, deets, itemsSourcedFromBase, err := bu.BackupCollections(
kopiaStats, deets, itemsSourcedFromBase, err := bc.ConsumeBackupCollections(
ctx,
bases,
cs,
@ -663,11 +686,11 @@ func (op *BackupOperation) persistResults(
return clues.New("backup population never completed")
}
if op.Status != Failed && opStats.gc.Metrics.Successes == 0 {
if op.Status != Failed && opStats.gc.IsZero() {
op.Status = NoData
}
op.Results.ItemsRead = opStats.gc.Metrics.Successes
op.Results.ItemsRead = opStats.gc.Successes
return op.Errors.Failure()
}
@ -714,8 +737,8 @@ func (op *BackupOperation) createBackupModels(
op.Status.String(),
backupID,
op.Selectors,
op.ResourceOwner,
op.ResourceOwnerName,
op.ResourceOwner.ID(),
op.ResourceOwner.Name(),
op.Results.ReadWrites,
op.Results.StartAndEndTime,
op.Errors.Errors())

View File

@ -152,7 +152,7 @@ func newTestBackupOp(
opts.ToggleFeatures = featureToggles
bo, err := NewBackupOperation(ctx, opts, kw, sw, gc, acct, sel, sel.DiscreteOwner, bus)
bo, err := NewBackupOperation(ctx, opts, kw, sw, gc, acct, sel, sel, bus)
if !assert.NoError(t, err, clues.ToCore(err)) {
closer()
t.FailNow()
@ -383,7 +383,7 @@ func generateContainerOfItems(
dest,
collections)
deets, err := gc.RestoreDataCollections(
deets, err := gc.ConsumeRestoreCollections(
ctx,
backupVersion,
acct,
@ -394,7 +394,9 @@ func generateContainerOfItems(
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
gc.AwaitStatus()
// have to wait here, both to ensure the process
// finishes, and also to clean up the gc status
gc.Wait()
return deets
}
@ -539,7 +541,7 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() {
func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
kw := &kopia.Wrapper{}
sw := &store.Wrapper{}
gc := &connector.GraphConnector{}
gc := &mockconnector.GraphConnector{}
acct := tester.NewM365Account(suite.T())
table := []struct {
@ -547,7 +549,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
opts control.Options
kw *kopia.Wrapper
sw *store.Wrapper
gc *connector.GraphConnector
bp BackupProducer
acct account.Account
targets []string
errCheck assert.ErrorAssertionFunc
@ -555,22 +557,24 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
{"good", control.Options{}, kw, sw, gc, acct, nil, assert.NoError},
{"missing kopia", control.Options{}, nil, sw, gc, acct, nil, assert.Error},
{"missing modelstore", control.Options{}, kw, nil, gc, acct, nil, assert.Error},
{"missing graphconnector", control.Options{}, kw, sw, nil, acct, nil, assert.Error},
{"missing backup producer", control.Options{}, kw, sw, nil, acct, nil, assert.Error},
}
for _, test := range table {
suite.Run(test.name, func() {
ctx, flush := tester.NewContext()
defer flush()
sel := selectors.Selector{DiscreteOwner: "test"}
_, err := NewBackupOperation(
ctx,
test.opts,
test.kw,
test.sw,
test.gc,
test.bp,
test.acct,
selectors.Selector{DiscreteOwner: "test"},
"test-name",
sel,
sel,
evmock.NewBus())
test.errCheck(suite.T(), err, clues.ToCore(err))
})

View File

@ -14,8 +14,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/data"
evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia"
@ -85,9 +84,9 @@ func checkPaths(t *testing.T, expected, got []path.Path) {
assert.ElementsMatch(t, expected, got)
}
// ----- backup producer
// ----- backup consumer
type mockBackuper struct {
type mockBackupConsumer struct {
checkFunc func(
bases []kopia.IncrementalBase,
cs []data.BackupCollection,
@ -95,7 +94,7 @@ type mockBackuper struct {
buildTreeWithBase bool)
}
func (mbu mockBackuper) BackupCollections(
func (mbu mockBackupConsumer) ConsumeBackupCollections(
ctx context.Context,
bases []kopia.IncrementalBase,
cs []data.BackupCollection,
@ -360,7 +359,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
var (
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
gc = &connector.GraphConnector{}
gc = &mockconnector.GraphConnector{}
acct = account.Account{}
now = time.Now()
)
@ -381,9 +380,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
TotalHashedBytes: 1,
TotalUploadedBytes: 1,
},
gc: &support.ConnectorOperationStatus{
Metrics: support.CollectionMetrics{Successes: 1},
},
gc: &data.CollectionStats{Successes: 1},
},
},
{
@ -392,7 +389,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
fail: assert.AnError,
stats: backupStats{
k: &kopia.BackupStats{},
gc: &support.ConnectorOperationStatus{},
gc: &data.CollectionStats{},
},
},
{
@ -400,7 +397,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
expectErr: assert.NoError,
stats: backupStats{
k: &kopia.BackupStats{},
gc: &support.ConnectorOperationStatus{},
gc: &data.CollectionStats{},
},
},
}
@ -418,7 +415,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
gc,
acct,
sel,
sel.DiscreteOwner,
sel,
evmock.NewBus())
require.NoError(t, err, clues.ToCore(err))
@ -427,7 +424,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
test.expectErr(t, op.persistResults(now, &test.stats))
assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status")
assert.Equal(t, test.stats.gc.Metrics.Successes, op.Results.ItemsRead, "items read")
assert.Equal(t, test.stats.gc.Successes, op.Results.ItemsRead, "items read")
assert.Equal(t, test.stats.k.TotalFileCount, op.Results.ItemsWritten, "items written")
assert.Equal(t, test.stats.k.TotalHashedBytes, op.Results.BytesRead, "bytes read")
assert.Equal(t, test.stats.k.TotalUploadedBytes, op.Results.BytesUploaded, "bytes written")
@ -564,7 +561,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
ctx, flush := tester.NewContext()
defer flush()
mbu := &mockBackuper{
mbu := &mockBackupConsumer{
checkFunc: func(
bases []kopia.IncrementalBase,
cs []data.BackupCollection,
@ -576,7 +573,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
}
//nolint:errcheck
consumeBackupDataCollections(
consumeBackupCollections(
ctx,
mbu,
tenant,

View File

@ -5,7 +5,6 @@ import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/pkg/control"
@ -57,7 +56,6 @@ type operation struct {
bus events.Eventer
kopia *kopia.Wrapper
store *store.Wrapper
gc *connector.GraphConnector
}
func newOperation(
@ -65,7 +63,6 @@ func newOperation(
bus events.Eventer,
kw *kopia.Wrapper,
sw *store.Wrapper,
gc *connector.GraphConnector,
) operation {
return operation{
CreatedAt: time.Now(),
@ -75,7 +72,6 @@ func newOperation(
bus: bus,
kopia: kw,
store: sw,
gc: gc,
Status: InProgress,
}
@ -90,9 +86,5 @@ func (op operation) validate() error {
return clues.New("missing modelstore")
}
if op.gc == nil {
return clues.New("missing graph connector")
}
return nil
}

View File

@ -8,7 +8,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/tester"
@ -26,30 +25,27 @@ func TestOperationSuite(t *testing.T) {
func (suite *OperationSuite) TestNewOperation() {
t := suite.T()
op := newOperation(control.Options{}, events.Bus{}, nil, nil, nil)
op := newOperation(control.Options{}, events.Bus{}, nil, nil)
assert.Greater(t, op.CreatedAt, time.Time{})
}
func (suite *OperationSuite) TestOperation_Validate() {
kwStub := &kopia.Wrapper{}
swStub := &store.Wrapper{}
gcStub := &connector.GraphConnector{}
table := []struct {
name string
kw *kopia.Wrapper
sw *store.Wrapper
gc *connector.GraphConnector
errCheck assert.ErrorAssertionFunc
}{
{"good", kwStub, swStub, gcStub, assert.NoError},
{"missing kopia wrapper", nil, swStub, gcStub, assert.Error},
{"missing store wrapper", kwStub, nil, gcStub, assert.Error},
{"missing graph connector", kwStub, swStub, nil, assert.Error},
{"good", kwStub, swStub, assert.NoError},
{"missing kopia wrapper", nil, swStub, assert.Error},
{"missing store wrapper", kwStub, nil, assert.Error},
}
for _, test := range table {
suite.Run(test.name, func() {
err := newOperation(control.Options{}, events.Bus{}, test.kw, test.sw, test.gc).validate()
err := newOperation(control.Options{}, events.Bus{}, test.kw, test.sw).validate()
test.errCheck(suite.T(), err, clues.ToCore(err))
})
}

View File

@ -7,12 +7,11 @@ import (
"github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/crash"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/events"
@ -42,6 +41,7 @@ type RestoreOperation struct {
Version string `json:"version"`
account account.Account
rc RestoreConsumer
}
// RestoreResults aggregate the details of the results of the operation.
@ -56,7 +56,7 @@ func NewRestoreOperation(
opts control.Options,
kw *kopia.Wrapper,
sw *store.Wrapper,
gc *connector.GraphConnector,
rc RestoreConsumer,
acct account.Account,
backupID model.StableID,
sel selectors.Selector,
@ -64,12 +64,13 @@ func NewRestoreOperation(
bus events.Eventer,
) (RestoreOperation, error) {
op := RestoreOperation{
operation: newOperation(opts, bus, kw, sw, gc),
operation: newOperation(opts, bus, kw, sw),
BackupID: backupID,
Selectors: sel,
Destination: dest,
Version: "v0",
account: acct,
rc: rc,
}
if err := op.validate(); err != nil {
return RestoreOperation{}, err
@ -79,6 +80,10 @@ func NewRestoreOperation(
}
func (op RestoreOperation) validate() error {
if op.rc == nil {
return clues.New("missing restore consumer")
}
return op.operation.validate()
}
@ -88,7 +93,7 @@ func (op RestoreOperation) validate() error {
// get populated asynchronously.
type restoreStats struct {
cs []data.RestoreCollection
gc *support.ConnectorOperationStatus
gc *data.CollectionStats
bytesRead *stats.ByteCounter
resourceCount int
@ -235,12 +240,9 @@ func (op *RestoreOperation) do(
opStats.resourceCount = 1
opStats.cs = dcs
restoreComplete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Restoring data"))
defer closer()
defer close(restoreComplete)
restoreDetails, err := op.gc.RestoreDataCollections(
deets, err = consumeRestoreCollections(
ctx,
op.rc,
bup.Version,
op.account,
op.Selectors,
@ -252,13 +254,11 @@ func (op *RestoreOperation) do(
return nil, clues.Wrap(err, "restoring collections")
}
restoreComplete <- struct{}{}
opStats.gc = op.rc.Wait()
opStats.gc = op.gc.AwaitStatus()
logger.Ctx(ctx).Debug(opStats.gc)
logger.Ctx(ctx).Debug(op.gc.PrintableStatus())
return restoreDetails, nil
return deets, nil
}
// persists details and statistics about the restore operation.
@ -285,11 +285,11 @@ func (op *RestoreOperation) persistResults(
return clues.New("restoration never completed")
}
if op.Status != Failed && opStats.gc.Metrics.Successes == 0 {
if op.Status != Failed && opStats.gc.IsZero() {
op.Status = NoData
}
op.Results.ItemsWritten = opStats.gc.Metrics.Successes
op.Results.ItemsWritten = opStats.gc.Successes
op.bus.Event(
ctx,
@ -312,6 +312,60 @@ func (op *RestoreOperation) persistResults(
return op.Errors.Failure()
}
// ---------------------------------------------------------------------------
// Restorer funcs
// ---------------------------------------------------------------------------
type RestoreConsumer interface {
ConsumeRestoreCollections(
ctx context.Context,
backupVersion int,
acct account.Account,
selector selectors.Selector,
dest control.RestoreDestination,
opts control.Options,
dcs []data.RestoreCollection,
errs *fault.Bus,
) (*details.Details, error)
// TODO: ConnectorOperationStatus should be replaced with something
// more generic.
Wait() *data.CollectionStats
}
func consumeRestoreCollections(
ctx context.Context,
rc RestoreConsumer,
backupVersion int,
acct account.Account,
sel selectors.Selector,
dest control.RestoreDestination,
opts control.Options,
dcs []data.RestoreCollection,
errs *fault.Bus,
) (*details.Details, error) {
complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Restoring data"))
defer func() {
complete <- struct{}{}
close(complete)
closer()
}()
deets, err := rc.ConsumeRestoreCollections(
ctx,
backupVersion,
acct,
sel,
dest,
opts,
dcs,
errs)
if err != nil {
return nil, errors.Wrap(err, "restoring collections")
}
return deets, nil
}
// formatDetailsForRestoration reduces the provided detail entries according to the
// selector specifications.
func formatDetailsForRestoration(

View File

@ -16,7 +16,6 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -50,7 +49,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
var (
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
gc = &connector.GraphConnector{}
gc = &mockconnector.GraphConnector{}
acct = account.Account{}
now = time.Now()
dest = tester.DefaultTestRestoreDestination()
@ -75,11 +74,9 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
Collection: &mockconnector.MockExchangeDataCollection{},
},
},
gc: &support.ConnectorOperationStatus{
Metrics: support.CollectionMetrics{
Objects: 1,
Successes: 1,
},
gc: &data.CollectionStats{
Objects: 1,
Successes: 1,
},
},
},
@ -89,7 +86,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
fail: assert.AnError,
stats: restoreStats{
bytesRead: &stats.ByteCounter{},
gc: &support.ConnectorOperationStatus{},
gc: &data.CollectionStats{},
},
},
{
@ -98,7 +95,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
stats: restoreStats{
bytesRead: &stats.ByteCounter{},
cs: []data.RestoreCollection{},
gc: &support.ConnectorOperationStatus{},
gc: &data.CollectionStats{},
},
},
}
@ -126,7 +123,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status")
assert.Equal(t, len(test.stats.cs), op.Results.ItemsRead, "items read")
assert.Equal(t, test.stats.gc.Metrics.Successes, op.Results.ItemsWritten, "items written")
assert.Equal(t, test.stats.gc.Successes, op.Results.ItemsWritten, "items written")
assert.Equal(t, test.stats.bytesRead.NumBytes, op.Results.BytesRead, "resource owners")
assert.Equal(t, test.stats.resourceCount, op.Results.ResourceOwners, "resource owners")
assert.Equal(t, now, op.Results.StartedAt, "started at")
@ -217,7 +214,7 @@ func (suite *RestoreOpIntegrationSuite) TearDownSuite() {
func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
kw := &kopia.Wrapper{}
sw := &store.Wrapper{}
gc := &connector.GraphConnector{}
gc := &mockconnector.GraphConnector{}
acct := tester.NewM365Account(suite.T())
dest := tester.DefaultTestRestoreDestination()
@ -226,7 +223,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
opts control.Options
kw *kopia.Wrapper
sw *store.Wrapper
gc *connector.GraphConnector
rc RestoreConsumer
acct account.Account
targets []string
errCheck assert.ErrorAssertionFunc
@ -234,7 +231,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
{"good", control.Options{}, kw, sw, gc, acct, nil, assert.NoError},
{"missing kopia", control.Options{}, nil, sw, gc, acct, nil, assert.Error},
{"missing modelstore", control.Options{}, kw, nil, gc, acct, nil, assert.Error},
{"missing graphConnector", control.Options{}, kw, sw, nil, acct, nil, assert.Error},
{"missing restore consumer", control.Options{}, kw, sw, nil, acct, nil, assert.Error},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -246,7 +243,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
test.opts,
test.kw,
test.sw,
test.gc,
test.rc,
test.acct,
"backup-id",
selectors.Selector{DiscreteOwner: "test"},
@ -295,7 +292,7 @@ func setupExchangeBackup(
gc,
acct,
bsel.Selector,
bsel.Selector.DiscreteOwner,
bsel.Selector,
evmock.NewBus())
require.NoError(t, err, clues.ToCore(err))
@ -352,7 +349,7 @@ func setupSharePointBackup(
gc,
acct,
spsel.Selector,
spsel.Selector.DiscreteOwner,
spsel.Selector,
evmock.NewBus())
require.NoError(t, err, clues.ToCore(err))

View File

@ -222,7 +222,7 @@ func collect(
}
type backuper interface {
BackupCollections(
ConsumeBackupCollections(
ctx context.Context,
bases []kopia.IncrementalBase,
cs []data.BackupCollection,
@ -240,7 +240,7 @@ func write(
dbcs []data.BackupCollection,
errs *fault.Bus,
) (string, error) {
backupStats, _, _, err := bup.BackupCollections(
backupStats, _, _, err := bup.ConsumeBackupCollections(
ctx,
nil,
dbcs,

View File

@ -24,6 +24,7 @@ const (
// M365 config
TestCfgAzureTenantID = "azure_tenantid"
TestCfgSiteID = "m365siteid"
TestCfgSiteURL = "m365siteurl"
TestCfgUserID = "m365userid"
TestCfgSecondaryUserID = "secondarym365userid"
TestCfgLoadTestUserID = "loadtestm365userid"
@ -34,6 +35,7 @@ const (
// test specific env vars
const (
EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID"
EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL"
EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID"
EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID"
EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID"
@ -136,6 +138,12 @@ func readTestConfig() (map[string]string, error) {
os.Getenv(EnvCorsoM365TestSiteID),
vpr.GetString(TestCfgSiteID),
"10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be")
fallbackTo(
testEnv,
TestCfgSiteURL,
os.Getenv(EnvCorsoM365TestSiteURL),
vpr.GetString(TestCfgSiteURL),
"https://10rqc2.sharepoint.com/sites/CorsoCI")
testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath)
testConfig = testEnv

View File

@ -52,7 +52,6 @@ func LoadTestM365SiteID(t *testing.T) string {
cfg, err := readTestConfig()
require.NoError(t, err, "retrieving load test m365 site id from test configuration", clues.ToCore(err))
// TODO: load test site id, not standard test site id
return cfg[TestCfgSiteID]
}
@ -133,3 +132,14 @@ func M365SiteID(t *testing.T) string {
return cfg[TestCfgSiteID]
}
// M365SiteURL returns a site webURL string representing the m365SiteURL described
// by either the env var CORSO_M365_TEST_SITE_URL, the corso_test.toml config
// file or the default value (in that order of priority). The default is a
// last-attempt fallback that will only work on alcion's testing org.
func M365SiteURL(t *testing.T) string {
cfg, err := readTestConfig()
require.NoError(t, err, "retrieving m365 site url from test configuration", clues.ToCore(err))
return cfg[TestCfgSiteURL]
}

View File

@ -165,6 +165,11 @@ type Printable struct {
// MinimumPrintable reduces the Backup to its minimally printable details.
func (b Backup) MinimumPrintable() any {
name := b.ResourceOwnerName
if len(name) == 0 {
name = b.Selector.Name()
}
return Printable{
ID: b.ID,
ErrorCount: b.ErrorCount,
@ -173,7 +178,7 @@ func (b Backup) MinimumPrintable() any {
Version: "0",
BytesRead: b.BytesRead,
BytesUploaded: b.BytesUploaded,
Owner: b.Selector.DiscreteOwner,
Owner: name,
}
}
@ -232,10 +237,15 @@ func (b Backup) Values() []string {
status += (")")
}
name := b.ResourceOwnerName
if len(name) == 0 {
name = b.Selector.Name()
}
return []string{
common.FormatTabularDisplayTime(b.StartedAt),
string(b.ID),
status,
b.Selector.DiscreteOwner,
name,
}
}

View File

@ -24,8 +24,10 @@ func TestBackupUnitSuite(t *testing.T) {
suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)})
}
const testDomainOwner = "test-domain-owner"
func stubBackup(t time.Time) backup.Backup {
sel := selectors.NewExchangeBackup([]string{"test"})
sel := selectors.NewExchangeBackup([]string{testDomainOwner})
sel.Include(sel.AllData())
return backup.Backup{
@ -61,30 +63,41 @@ func stubBackup(t time.Time) backup.Backup {
func (suite *BackupUnitSuite) TestBackup_HeadersValues() {
var (
t = suite.T()
now = time.Now()
b = stubBackup(now)
expectHs = []string{
"Started At",
"ID",
"Status",
"Resource Owner",
}
nowFmt = common.FormatTabularDisplayTime(now)
expectVs = []string{
nowFmt,
"id",
"status (2 errors, 1 skipped: 1 malware)",
"test",
}
now = time.Now()
nowFmt = common.FormatTabularDisplayTime(now)
expectHs = []string{"Started At", "ID", "Status", "Resource Owner"}
expectVsBase = []string{nowFmt, "id", "status (2 errors, 1 skipped: 1 malware)"}
)
// single skipped malware
hs := b.Headers()
assert.Equal(t, expectHs, hs)
table := []struct {
name string
bup func() backup.Backup
expectVs []string
}{
{
name: "owner from selectors",
bup: func() backup.Backup { return stubBackup(now) },
expectVs: append(expectVsBase, testDomainOwner),
},
{
name: "owner from backup",
bup: func() backup.Backup {
b := stubBackup(now)
b.ResourceOwnerName = "test ro name"
return b
},
expectVs: append(expectVsBase, "test ro name"),
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
b := test.bup()
vs := b.Values()
assert.Equal(t, expectVs, vs)
assert.Equal(t, expectHs, b.Headers()) // not elementsMatch, order matters
assert.Equal(t, test.expectVs, b.Values()) // not elementsMatch, order matters
})
}
}
func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {

View File

@ -359,21 +359,23 @@ func newSliceFilter(c comparator, targets, normTargets []string, negate bool) Fi
// ----------------------------------------------------------------------------------------------------
// CompareAny checks whether any one of all the provided
// inputs passes the filter.
// inputs passes the filter. If one passes, that value is
// returned, as well as its index in the input range.
// If nothing matches, returns ("", -1, false)
//
// Note that, as a gotcha, CompareAny can resolve truthily
// for both the standard and negated versions of a filter.
// Ex: consider the input CompareAny(true, false), which
// will return true for both Equals(true) and NotEquals(true),
// because at least one element matches for both filters.
func (f Filter) CompareAny(inputs ...string) bool {
for _, in := range inputs {
func (f Filter) CompareAny(inputs ...string) (string, int, bool) {
for i, in := range inputs {
if f.Compare(in) {
return true
return in, i, true
}
}
return false
return "", -1, false
}
// Compare checks whether the input passes the filter.
@ -442,6 +444,48 @@ func (f Filter) Compare(input string) bool {
return res
}
// Matches extends Compare by not only checking if
// the input passes the filter, but if it passes, the
// target which matched and its index are returned as well.
// If more than one value matches the input, only the
// first is returned.
// returns ("", -1, false) if no match is found.
// TODO: only partially implemented.
// func (f Filter) Matches(input string) (string, int, bool) {
// var (
// cmp func(string, string) bool
// res bool
// targets = f.NormalizedTargets
// )
// switch f.Comparator {
// case TargetPathPrefix:
// cmp = pathPrefix
// case TargetPathContains:
// cmp = pathContains
// case TargetPathSuffix:
// cmp = pathSuffix
// case TargetPathEquals:
// cmp = pathEquals
// default:
// return "", -1, false
// }
// for i, tgt := range targets {
// res = cmp(norm(tgt), norm(input))
// if !f.Negate && res {
// return f.Targets[i], i, true
// }
// if f.Negate && !res {
// return f.Targets[i], i, true
// }
// }
// return "", -1, false
// }
// true if t == i
func equals(target, input string) bool {
return target == input

View File

@ -45,20 +45,49 @@ func (suite *FiltersSuite) TestEquals_any() {
nf := filters.NotEqual("foo")
table := []struct {
name string
input []string
expectF assert.BoolAssertionFunc
expectNF assert.BoolAssertionFunc
name string
input []string
expectF assert.BoolAssertionFunc
expectFVal string
expectFIdx int
expectNF assert.BoolAssertionFunc
expectNFVal string
expectNFIdx int
}{
{"includes target", []string{"foo", "bar"}, assert.True, assert.True},
{"not includes target", []string{"baz", "qux"}, assert.False, assert.True},
{
name: "includes target",
input: []string{"foo", "bar"},
expectF: assert.True,
expectFVal: "foo",
expectFIdx: 0,
expectNF: assert.True,
expectNFVal: "bar",
expectNFIdx: 1,
},
{
name: "not includes target",
input: []string{"baz", "qux"},
expectF: assert.False,
expectFVal: "",
expectFIdx: -1,
expectNF: assert.True,
expectNFVal: "baz",
expectNFIdx: 0,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
test.expectF(t, f.CompareAny(test.input...), "filter")
test.expectNF(t, nf.CompareAny(test.input...), "negated filter")
v, i, b := f.CompareAny(test.input...)
test.expectF(t, b, "filter")
assert.Equal(t, test.expectFIdx, i, "index")
assert.Equal(t, test.expectFVal, v, "value")
v, i, b = nf.CompareAny(test.input...)
test.expectNF(t, b, "neg-filter")
assert.Equal(t, test.expectNFIdx, i, "neg-index")
assert.Equal(t, test.expectNFVal, v, "neg-value")
})
}
}

View File

@ -120,7 +120,7 @@ func runLoadTest(
) {
//revive:enable:context-as-argument
t.Run(prefix+"_load_test_main", func(t *testing.T) {
b, err := r.NewBackup(ctx, bupSel)
b, err := r.NewBackup(ctx, bupSel, nil, nil)
require.NoError(t, err, clues.ToCore(err))
runBackupLoadTest(t, ctx, &b, service, usersUnderTest)
@ -447,8 +447,7 @@ func (suite *LoadExchangeSuite) TestExchange() {
"all_users", "exchange",
suite.usersUnderTest,
sel, sel, // same selection for backup and restore
true,
)
true)
}
// single user, lots of data
@ -500,8 +499,7 @@ func (suite *IndividualLoadExchangeSuite) TestExchange() {
"single_user", "exchange",
suite.usersUnderTest,
sel, sel, // same selection for backup and restore
true,
)
true)
}
// ------------------------------------------------------------------------------------------------
@ -553,8 +551,7 @@ func (suite *LoadOneDriveSuite) TestOneDrive() {
"all_users", "one_drive",
suite.usersUnderTest,
sel, sel, // same selection for backup and restore
false,
)
false)
}
type IndividualLoadOneDriveSuite struct {
@ -601,8 +598,7 @@ func (suite *IndividualLoadOneDriveSuite) TestOneDrive() {
"single_user", "one_drive",
suite.usersUnderTest,
sel, sel, // same selection for backup and restore
false,
)
false)
}
// ------------------------------------------------------------------------------------------------
@ -654,8 +650,7 @@ func (suite *LoadSharePointSuite) TestSharePoint() {
"all_sites", "share_point",
suite.sitesUnderTest,
sel, sel, // same selection for backup and restore
false,
)
false)
}
type IndividualLoadSharePointSuite struct {
@ -703,6 +698,5 @@ func (suite *IndividualLoadSharePointSuite) TestSharePoint() {
"single_site", "share_point",
suite.sitesUnderTest,
sel, sel, // same selection for backup and restore
false,
)
false)
}

View File

@ -54,6 +54,7 @@ type Repository interface {
NewBackup(
ctx context.Context,
self selectors.Selector,
ownerIDToName, ownerNameToID map[string]string,
) (operations.BackupOperation, error)
NewRestore(
ctx context.Context,
@ -283,15 +284,30 @@ func (r *repository) Close(ctx context.Context) error {
}
// NewBackup generates a BackupOperation runner.
// ownerIDToName and ownerNameToID are optional populations, in case the caller has
// already generated those values.
func (r repository) NewBackup(
ctx context.Context,
sel selectors.Selector,
ownerIDToName, ownerNameToID map[string]string,
) (operations.BackupOperation, error) {
gc, err := connectToM365(ctx, sel, r.Account, fault.New(true))
if err != nil {
return operations.BackupOperation{}, errors.Wrap(err, "connecting to m365")
}
ownerID, ownerName, err := gc.PopulateOwnerIDAndNamesFrom(
ctx,
sel.DiscreteOwner,
ownerIDToName,
ownerNameToID)
if err != nil {
return operations.BackupOperation{}, errors.Wrap(err, "resolving resource owner details")
}
// TODO: retrieve display name from gc
sel = sel.SetDiscreteOwnerIDName(ownerID, ownerName)
return operations.NewBackupOperation(
ctx,
r.Opts,
@ -300,7 +316,7 @@ func (r repository) NewBackup(
gc,
r.Account,
sel,
sel.DiscreteOwner,
sel,
r.Bus)
}

View File

@ -198,7 +198,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err, clues.ToCore(err))
bo, err := r.NewBackup(ctx, selectors.Selector{DiscreteOwner: "test"})
bo, err := r.NewBackup(ctx, selectors.Selector{DiscreteOwner: "test"}, nil, nil)
require.NoError(t, err, clues.ToCore(err))
require.NotNil(t, bo)
}

View File

@ -223,7 +223,9 @@ func matchesAny[T scopeT, C categoryT](s T, cat C, inpts []string) bool {
return false
}
return s[cat.String()].CompareAny(inpts...)
_, _, pass := s[cat.String()].CompareAny(inpts...)
return pass
}
// getCategory returns the scope's category value.

View File

@ -112,6 +112,8 @@ type Selector struct {
// iterate over the results, where each one will populate this field
// with a different owner.
DiscreteOwner string `json:"discreteOwner,omitempty"`
// display name for the DiscreteOwner.
DiscreteOwnerName string `json:"discreteOwnerName,omitempty"`
// A slice of exclusion scopes. Exclusions apply globally to all
// inclusions/filters, with any-match behavior.
@ -146,6 +148,46 @@ func (s Selector) DiscreteResourceOwners() []string {
return split(s.ResourceOwners.Target)
}
// SetDiscreteOwnerIDName ensures the selector has the correct discrete owner
// id and name. It is assumed that these values are sourced using the current
// s.DiscreteOwner as input. The reason for taking in both the id and name, and
// not just the name, is so that constructors can input owner aliases in place
// of ids, with the expectation that the two will get sorted and re-written
// later on with this setter.
func (s Selector) SetDiscreteOwnerIDName(id, name string) Selector {
r := s
if len(id) == 0 {
// assume a the discreteOwner is already set, and don't replace anything.
r.DiscreteOwnerName = s.DiscreteOwner
return r
}
r.DiscreteOwner = id
r.DiscreteOwnerName = name
if len(name) == 0 {
r.DiscreteOwnerName = id
}
return r
}
// ID returns s.discreteOwner, which is assumed to be a stable ID.
func (s Selector) ID() string {
return s.DiscreteOwner
}
// Name returns s.discreteOwnerName. If that value is empty, it returns
// s.DiscreteOwner instead.
func (s Selector) Name() string {
if len(s.DiscreteOwnerName) == 0 {
return s.DiscreteOwner
}
return s.DiscreteOwnerName
}
// isAnyResourceOwner returns true if the selector includes all resource owners.
func isAnyResourceOwner(s Selector) bool {
return s.ResourceOwners.Comparator == filters.Passes
@ -336,7 +378,7 @@ func pathCategoriesIn[T scopeT, C categoryT](ss []scope) []path.CategoryType {
}
// ---------------------------------------------------------------------------
// scope helpers
// scope constructors
// ---------------------------------------------------------------------------
type scopeConfig struct {

View File

@ -248,6 +248,47 @@ func (suite *SelectorSuite) TestSplitByResourceOnwer() {
}
}
func (suite *SelectorSuite) TestIDName() {
table := []struct {
title string
id, name string
expectID, expectName string
}{
{"empty", "", "", "", ""},
{"only id", "id", "", "id", "id"},
{"only name", "", "name", "", "name"},
{"both", "id", "name", "id", "name"},
}
for _, test := range table {
suite.Run(test.title, func() {
sel := Selector{DiscreteOwner: test.id, DiscreteOwnerName: test.name}
assert.Equal(suite.T(), test.expectID, sel.ID())
assert.Equal(suite.T(), test.expectName, sel.Name())
})
}
}
func (suite *SelectorSuite) TestSetDiscreteOwnerIDName() {
table := []struct {
title string
id, name string
expectID, expectName string
}{
{"empty", "", "", "", ""},
{"only id", "id", "", "id", "id"},
{"only name", "", "", "", ""},
{"both", "id", "name", "id", "name"},
}
for _, test := range table {
suite.Run(test.title, func() {
sel := Selector{}
sel = sel.SetDiscreteOwnerIDName(test.id, test.name)
assert.Equal(suite.T(), test.expectID, sel.ID())
assert.Equal(suite.T(), test.expectName, sel.Name())
})
}
}
// TestPathCategories verifies that no scope produces a `path.UnknownCategory`
func (suite *SelectorSuite) TestPathCategories_includes() {
users := []string{"someuser@onmicrosoft.com"}

View File

@ -6,9 +6,8 @@ import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/discovery"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -55,84 +54,6 @@ func Users(ctx context.Context, acct account.Account, errs *fault.Bus) ([]*User,
return ret, nil
}
func UserIDs(ctx context.Context, acct account.Account, errs *fault.Bus) ([]string, error) {
users, err := Users(ctx, acct, errs)
if err != nil {
return nil, err
}
ret := make([]string, 0, len(users))
for _, u := range users {
ret = append(ret, u.ID)
}
return ret, nil
}
// UserPNs retrieves all user principleNames in the tenant. Principle Names
// can be used analogous userIDs in graph API queries.
func UserPNs(ctx context.Context, acct account.Account, errs *fault.Bus) ([]string, error) {
users, err := Users(ctx, acct, errs)
if err != nil {
return nil, err
}
ret := make([]string, 0, len(users))
for _, u := range users {
ret = append(ret, u.PrincipalName)
}
return ret, nil
}
type Site struct {
// WebURL that displays the item in the browser
WebURL string
// ID is of the format: <site collection hostname>.<site collection unique id>.<site unique id>
// for example: contoso.sharepoint.com,abcdeab3-0ccc-4ce1-80ae-b32912c9468d,xyzud296-9f7c-44e1-af81-3c06d0d43007
ID string
}
// Sites returns a list of Sites in a specified M365 tenant
func Sites(ctx context.Context, acct account.Account, errs *fault.Bus) ([]*Site, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites, errs)
if err != nil {
return nil, clues.Wrap(err, "initializing M365 graph connection")
}
// gc.Sites is a map with keys: SiteURL, values: ID
ret := make([]*Site, 0, len(gc.Sites))
for k, v := range gc.Sites {
ret = append(ret, &Site{
WebURL: k,
ID: v,
})
}
return ret, nil
}
// SiteURLs returns a list of SharePoint site WebURLs in the specified M365 tenant
func SiteURLs(ctx context.Context, acct account.Account, errs *fault.Bus) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites, errs)
if err != nil {
return nil, clues.Wrap(err, "initializing M365 graph connection")
}
return gc.GetSiteWebURLs(), nil
}
// SiteIDs returns a list of SharePoint sites IDs in the specified M365 tenant
func SiteIDs(ctx context.Context, acct account.Account, errs *fault.Bus) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites, errs)
if err != nil {
return nil, clues.Wrap(err, "initializing graph connection")
}
return gc.GetSiteIDs(), nil
}
// parseUser extracts information from `models.Userable` we care about
func parseUser(item models.Userable) (*User, error) {
if item.GetUserPrincipalName() == nil {
@ -148,3 +69,93 @@ func parseUser(item models.Userable) (*User, error) {
return u, nil
}
// UsersMap retrieves all users in the tenant, and returns two maps: one id-to-principalName,
// and one principalName-to-id.
func UsersMap(
ctx context.Context,
acct account.Account,
errs *fault.Bus,
) (map[string]string, map[string]string, error) {
users, err := Users(ctx, acct, errs)
if err != nil {
return nil, nil, err
}
var (
idToName = make(map[string]string, len(users))
nameToID = make(map[string]string, len(users))
)
for _, u := range users {
idToName[u.ID] = u.PrincipalName
nameToID[u.PrincipalName] = u.ID
}
return idToName, nameToID, nil
}
type Site struct {
// WebURL that displays the item in the browser
WebURL string
// ID is of the format: <site collection hostname>.<site collection unique id>.<site unique id>
// for example: contoso.sharepoint.com,abcdeab3-0ccc-4ce1-80ae-b32912c9468d,xyzud296-9f7c-44e1-af81-3c06d0d43007
ID string
}
// Sites returns a list of Sites in a specified M365 tenant
func Sites(ctx context.Context, acct account.Account, errs *fault.Bus) ([]*Site, error) {
sites, err := discovery.Sites(ctx, acct, errs)
if err != nil {
return nil, clues.Wrap(err, "initializing M365 graph connection")
}
ret := make([]*Site, 0, len(sites))
for _, s := range sites {
ps, err := parseSite(s)
if err != nil {
return nil, clues.Wrap(err, "parsing siteable")
}
ret = append(ret, ps)
}
return ret, nil
}
// parseSite extracts the information from `models.Siteable` we care about
func parseSite(item models.Siteable) (*Site, error) {
s := &Site{
ID: ptr.Val(item.GetId()),
WebURL: ptr.Val(item.GetWebUrl()),
}
return s, nil
}
// SitesMap retrieves all sites in the tenant, and returns two maps: one id-to-webURL,
// and one webURL-to-id.
func SitesMap(
ctx context.Context,
acct account.Account,
errs *fault.Bus,
) (map[string]string, map[string]string, error) {
sites, err := Sites(ctx, acct, errs)
if err != nil {
return nil, nil, err
}
var (
idToName = make(map[string]string, len(sites))
nameToID = make(map[string]string, len(sites))
)
for _, s := range sites {
idToName[s.ID] = s.WebURL
nameToID[s.WebURL] = s.ID
}
return idToName, nameToID, nil
}