e86592f51e
Now that graph errors are always transformed as part of the graph client wrapper or http_wrapper, we don't need to call graph.Stack or graph.Wrap outside of those inner helpers any longer. This PR swaps all those graph calls with equivalent clues funcs as a cleanup. Should not contain any logical changes. --- #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🧹 Tech Debt/Cleanup #### Issue(s) * #4685 #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
380 lines
8.6 KiB
Go
380 lines
8.6 KiB
Go
package site
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
stdpath "path"
|
|
"time"
|
|
|
|
"github.com/alcionai/clues"
|
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
|
|
|
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
|
"github.com/alcionai/corso/src/internal/data"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
|
betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
|
|
"github.com/alcionai/corso/src/internal/m365/support"
|
|
"github.com/alcionai/corso/src/internal/observe"
|
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
|
"github.com/alcionai/corso/src/pkg/account"
|
|
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
|
"github.com/alcionai/corso/src/pkg/count"
|
|
"github.com/alcionai/corso/src/pkg/fault"
|
|
"github.com/alcionai/corso/src/pkg/logger"
|
|
"github.com/alcionai/corso/src/pkg/path"
|
|
"github.com/alcionai/corso/src/pkg/selectors"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
|
)
|
|
|
|
// CollectLibraries constructs a onedrive Collections struct and Get()s
|
|
// all the drives associated with the site.
|
|
func CollectLibraries(
|
|
ctx context.Context,
|
|
bpc inject.BackupProducerConfig,
|
|
bh drive.BackupHandler,
|
|
tenantID string,
|
|
ssmb *prefixmatcher.StringSetMatchBuilder,
|
|
su support.StatusUpdater,
|
|
counter *count.Bus,
|
|
errs *fault.Bus,
|
|
) ([]data.BackupCollection, bool, error) {
|
|
logger.Ctx(ctx).Debug("creating SharePoint Library collections")
|
|
|
|
var (
|
|
collections = []data.BackupCollection{}
|
|
colls = drive.NewCollections(
|
|
bh,
|
|
tenantID,
|
|
bpc.ProtectedResource,
|
|
su,
|
|
bpc.Options,
|
|
counter)
|
|
)
|
|
|
|
msg := fmt.Sprintf(
|
|
"%s (%s)",
|
|
path.LibrariesCategory.HumanString(),
|
|
stdpath.Base(bpc.ProtectedResource.Name()))
|
|
|
|
progressMessage := observe.MessageWithCompletion(
|
|
ctx,
|
|
observe.ProgressCfg{
|
|
Indent: 1,
|
|
CompletionMessage: func() string { return fmt.Sprintf("(found %d items)", colls.NumItems) },
|
|
},
|
|
msg)
|
|
close(progressMessage)
|
|
|
|
odcs, canUsePreviousBackup, err := colls.Get(ctx, bpc.MetadataCollections, ssmb, errs)
|
|
if err != nil {
|
|
return nil, false, clues.Wrap(err, "getting library")
|
|
}
|
|
|
|
return append(collections, odcs...), canUsePreviousBackup, nil
|
|
}
|
|
|
|
// CollectPages constructs a sharepoint Collections struct and Get()s the associated
|
|
// M365 IDs for the associated Pages.
|
|
func CollectPages(
|
|
ctx context.Context,
|
|
bpc inject.BackupProducerConfig,
|
|
creds account.M365Config,
|
|
ac api.Client,
|
|
scope selectors.SharePointScope,
|
|
su support.StatusUpdater,
|
|
counter *count.Bus,
|
|
errs *fault.Bus,
|
|
) ([]data.BackupCollection, error) {
|
|
logger.Ctx(ctx).Debug("creating SharePoint Pages collections")
|
|
|
|
var (
|
|
el = errs.Local()
|
|
spcs = make([]data.BackupCollection, 0)
|
|
)
|
|
|
|
// make the betaClient
|
|
// Need to receive From DataCollection Call
|
|
adpt, err := graph.CreateAdapter(
|
|
creds.AzureTenantID,
|
|
creds.AzureClientID,
|
|
creds.AzureClientSecret,
|
|
counter)
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "creating azure client adapter")
|
|
}
|
|
|
|
betaService := betaAPI.NewBetaService(adpt)
|
|
|
|
tuples, err := betaAPI.FetchPages(ctx, betaService, bpc.ProtectedResource.ID())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, tuple := range tuples {
|
|
if el.Failure() != nil {
|
|
break
|
|
}
|
|
|
|
dir, err := path.Build(
|
|
creds.AzureTenantID,
|
|
bpc.ProtectedResource.ID(),
|
|
path.SharePointService,
|
|
path.PagesCategory,
|
|
false,
|
|
tuple.Name)
|
|
if err != nil {
|
|
el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "creating page collection path"))
|
|
}
|
|
|
|
collection := NewPrefetchCollection(
|
|
nil,
|
|
dir,
|
|
nil,
|
|
nil,
|
|
ac,
|
|
scope,
|
|
su,
|
|
bpc.Options,
|
|
nil)
|
|
collection.SetBetaService(betaService)
|
|
collection.AddItem(tuple.ID, time.Now())
|
|
|
|
spcs = append(spcs, collection)
|
|
}
|
|
|
|
return spcs, el.Failure()
|
|
}
|
|
|
|
func CollectLists(
|
|
ctx context.Context,
|
|
bh backupHandler,
|
|
bpc inject.BackupProducerConfig,
|
|
ac api.Client,
|
|
tenantID string,
|
|
scope selectors.SharePointScope,
|
|
su support.StatusUpdater,
|
|
counter *count.Bus,
|
|
errs *fault.Bus,
|
|
) ([]data.BackupCollection, bool, error) {
|
|
logger.Ctx(ctx).Debug("Creating SharePoint List Collections")
|
|
|
|
var (
|
|
el = errs.Local()
|
|
spcs = make([]data.BackupCollection, 0)
|
|
cfg = api.CallConfig{Select: idAnd("list", "lastModifiedDateTime")}
|
|
)
|
|
|
|
dps, canUsePreviousBackup, err := parseListsMetadataCollections(ctx, path.ListsCategory, bpc.MetadataCollections)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
ctx = clues.Add(ctx, "can_use_previous_backup", canUsePreviousBackup)
|
|
|
|
lists, err := bh.GetItems(ctx, cfg)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
collections, err := populateListsCollections(
|
|
ctx,
|
|
bh,
|
|
bpc,
|
|
ac,
|
|
tenantID,
|
|
scope,
|
|
su,
|
|
lists,
|
|
dps,
|
|
counter,
|
|
el)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
for _, spc := range collections {
|
|
spcs = append(spcs, spc)
|
|
}
|
|
|
|
return spcs, canUsePreviousBackup, el.Failure()
|
|
}
|
|
|
|
func populateListsCollections(
|
|
ctx context.Context,
|
|
bh backupHandler,
|
|
bpc inject.BackupProducerConfig,
|
|
ac api.Client,
|
|
tenantID string,
|
|
scope selectors.SharePointScope,
|
|
su support.StatusUpdater,
|
|
lists []models.Listable,
|
|
dps metadata.DeltaPaths,
|
|
counter *count.Bus,
|
|
el *fault.Bus,
|
|
) (map[string]data.BackupCollection, error) {
|
|
var (
|
|
err error
|
|
collection data.BackupCollection
|
|
// collections: list-id -> backup-collection
|
|
collections = make(map[string]data.BackupCollection)
|
|
currPaths = make(map[string]string)
|
|
tombstones = makeTombstones(dps)
|
|
)
|
|
|
|
counter.Add(count.Lists, int64(len(lists)))
|
|
|
|
for _, list := range lists {
|
|
if el.Failure() != nil {
|
|
break
|
|
}
|
|
|
|
if api.SkipListTemplates.HasKey(ptr.Val(list.GetList().GetTemplate())) {
|
|
continue
|
|
}
|
|
|
|
var (
|
|
listID = ptr.Val(list.GetId())
|
|
storageDir = path.Elements{listID}
|
|
dp = dps[storageDir.String()]
|
|
prevPathStr = dp.Path
|
|
prevPath path.Path
|
|
)
|
|
|
|
delete(tombstones, listID)
|
|
|
|
if len(prevPathStr) > 0 {
|
|
if prevPath, err = pathFromPrevString(prevPathStr); err != nil {
|
|
err = clues.StackWC(ctx, err).Label(count.BadPrevPath)
|
|
logger.CtxErr(ctx, err).Error("parsing prev path")
|
|
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
currPath, err := bh.CanonicalPath(storageDir, tenantID)
|
|
if err != nil {
|
|
el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "creating list collection path"))
|
|
return nil, err
|
|
}
|
|
|
|
modTime := ptr.Val(list.GetLastModifiedDateTime())
|
|
|
|
lazyFetchCol := NewLazyFetchCollection(
|
|
bh,
|
|
currPath,
|
|
prevPath,
|
|
storageDir.Builder(),
|
|
su,
|
|
counter.Local())
|
|
|
|
lazyFetchCol.AddItem(
|
|
ptr.Val(list.GetId()),
|
|
modTime)
|
|
|
|
collection = lazyFetchCol
|
|
|
|
// Always use lazyFetchCol.
|
|
// In case we receive zero mod time from graph fallback to prefetchCol.
|
|
if modTime.IsZero() {
|
|
prefetchCol := NewPrefetchCollection(
|
|
bh,
|
|
currPath,
|
|
prevPath,
|
|
storageDir.Builder(),
|
|
ac,
|
|
scope,
|
|
su,
|
|
bpc.Options,
|
|
counter.Local())
|
|
|
|
prefetchCol.AddItem(
|
|
ptr.Val(list.GetId()),
|
|
modTime)
|
|
|
|
collection = prefetchCol
|
|
}
|
|
|
|
collections[storageDir.String()] = collection
|
|
currPaths[storageDir.String()] = currPath.String()
|
|
}
|
|
|
|
handleTombstones(ctx, bpc, tombstones, collections, counter, el)
|
|
|
|
// Build metadata path
|
|
pathPrefix, err := path.BuildMetadata(
|
|
tenantID,
|
|
bpc.ProtectedResource.ID(),
|
|
path.SharePointService,
|
|
path.ListsCategory,
|
|
false)
|
|
if err != nil {
|
|
return nil, clues.WrapWC(ctx, err, "making metadata path prefix").
|
|
Label(count.BadPathPrefix)
|
|
}
|
|
|
|
mdCol, err := graph.MakeMetadataCollection(
|
|
pathPrefix,
|
|
[]graph.MetadataCollectionEntry{
|
|
graph.NewMetadataEntry(metadata.PreviousPathFileName, currPaths),
|
|
},
|
|
su,
|
|
counter.Local())
|
|
if err != nil {
|
|
return nil, clues.WrapWC(ctx, err, "making metadata collection")
|
|
}
|
|
|
|
collections["metadata"] = mdCol
|
|
|
|
return collections, nil
|
|
}
|
|
|
|
func idAnd(ss ...string) []string {
|
|
id := []string{"id"}
|
|
|
|
if len(ss) == 0 {
|
|
return id
|
|
}
|
|
|
|
return append(id, ss...)
|
|
}
|
|
|
|
func handleTombstones(
|
|
ctx context.Context,
|
|
bpc inject.BackupProducerConfig,
|
|
tombstones map[string]string,
|
|
collections map[string]data.BackupCollection,
|
|
counter *count.Bus,
|
|
el *fault.Bus,
|
|
) {
|
|
for id, p := range tombstones {
|
|
if el.Failure() != nil {
|
|
return
|
|
}
|
|
|
|
ictx := clues.Add(ctx, "tombstone_id", id)
|
|
|
|
if collections[id] != nil {
|
|
err := clues.NewWC(ictx, "conflict: tombstone exists for a live collection").Label(count.CollectionTombstoneConflict)
|
|
el.AddRecoverable(ictx, err)
|
|
|
|
continue
|
|
}
|
|
|
|
if len(p) == 0 {
|
|
continue
|
|
}
|
|
|
|
prevPath, err := pathFromPrevString(p)
|
|
if err != nil {
|
|
err := clues.StackWC(ictx, err).Label(count.BadPrevPath)
|
|
logger.CtxErr(ictx, err).Error("parsing tombstone prev path")
|
|
|
|
continue
|
|
}
|
|
|
|
collections[id] = data.NewTombstoneCollection(prevPath, bpc.Options, counter.Local())
|
|
}
|
|
}
|