9ecefd2569
Increase proliferation of the count bus to record runtime stats. --- #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🤖 Supportability/Tests #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
390 lines
9.3 KiB
Go
390 lines
9.3 KiB
Go
package groups
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/alcionai/clues"
|
|
"github.com/kopia/kopia/repo/manifest"
|
|
|
|
"github.com/alcionai/corso/src/internal/common/idname"
|
|
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
|
"github.com/alcionai/corso/src/internal/data"
|
|
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/groups"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/site"
|
|
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
|
|
"github.com/alcionai/corso/src/internal/m365/support"
|
|
"github.com/alcionai/corso/src/internal/observe"
|
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
|
"github.com/alcionai/corso/src/pkg/account"
|
|
"github.com/alcionai/corso/src/pkg/backup/identity"
|
|
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
|
"github.com/alcionai/corso/src/pkg/control"
|
|
"github.com/alcionai/corso/src/pkg/count"
|
|
"github.com/alcionai/corso/src/pkg/fault"
|
|
"github.com/alcionai/corso/src/pkg/logger"
|
|
"github.com/alcionai/corso/src/pkg/path"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
|
)
|
|
|
|
func ProduceBackupCollections(
|
|
ctx context.Context,
|
|
bpc inject.BackupProducerConfig,
|
|
ac api.Client,
|
|
creds account.M365Config,
|
|
su support.StatusUpdater,
|
|
counter *count.Bus,
|
|
errs *fault.Bus,
|
|
) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, error) {
|
|
b, err := bpc.Selector.ToGroupsBackup()
|
|
if err != nil {
|
|
return nil, nil, clues.Wrap(err, "groupsDataCollection: parsing selector")
|
|
}
|
|
|
|
var (
|
|
el = errs.Local()
|
|
collections = []data.BackupCollection{}
|
|
categories = map[path.CategoryType]struct{}{}
|
|
ssmb = prefixmatcher.NewStringSetBuilder()
|
|
sitesPreviousPaths = map[string]string{}
|
|
)
|
|
|
|
ctx = clues.Add(
|
|
ctx,
|
|
"group_id", clues.Hide(bpc.ProtectedResource.ID()),
|
|
"group_name", clues.Hide(bpc.ProtectedResource.Name()))
|
|
|
|
group, err := ac.Groups().GetByID(
|
|
ctx,
|
|
bpc.ProtectedResource.ID(),
|
|
api.CallConfig{})
|
|
if err != nil {
|
|
return nil, nil, clues.WrapWC(ctx, err, "getting group")
|
|
}
|
|
|
|
isTeam := api.IsTeam(ctx, group)
|
|
|
|
for _, scope := range b.Scopes() {
|
|
if el.Failure() != nil {
|
|
break
|
|
}
|
|
|
|
cl := counter.Local()
|
|
ictx := clues.AddLabelCounter(ctx, cl.PlainAdder())
|
|
|
|
var dbcs []data.BackupCollection
|
|
|
|
switch scope.Category().PathType() {
|
|
case path.LibrariesCategory:
|
|
sites, err := ac.Groups().GetAllSites(ictx, bpc.ProtectedResource.ID(), errs)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
cl.Add(count.Sites, int64(len(sites)))
|
|
|
|
siteMetadataCollection := map[string][]data.RestoreCollection{}
|
|
|
|
// Once we have metadata collections for chat as well, we will have to filter those out
|
|
for _, c := range bpc.MetadataCollections {
|
|
siteID := c.FullPath().Elements().Last()
|
|
siteMetadataCollection[siteID] = append(siteMetadataCollection[siteID], c)
|
|
}
|
|
|
|
for _, s := range sites {
|
|
var (
|
|
scl = cl.Local()
|
|
pr = idname.NewProvider(ptr.Val(s.GetId()), ptr.Val(s.GetWebUrl()))
|
|
sbpc = inject.BackupProducerConfig{
|
|
LastBackupVersion: bpc.LastBackupVersion,
|
|
Options: bpc.Options,
|
|
ProtectedResource: pr,
|
|
Selector: bpc.Selector,
|
|
MetadataCollections: siteMetadataCollection[ptr.Val(s.GetId())],
|
|
}
|
|
bh = drive.NewGroupBackupHandler(
|
|
bpc.ProtectedResource.ID(),
|
|
ptr.Val(s.GetId()),
|
|
ac.Drives(),
|
|
scope)
|
|
)
|
|
|
|
ictx = clues.Add(
|
|
ictx,
|
|
"site_id", ptr.Val(s.GetId()),
|
|
"site_weburl", graph.LoggableURL(ptr.Val(s.GetWebUrl())))
|
|
|
|
sp, err := bh.SitePathPrefix(creds.AzureTenantID)
|
|
if err != nil {
|
|
return nil, nil, clues.WrapWC(ictx, err, "getting site path").Label(count.BadPathPrefix)
|
|
}
|
|
|
|
sitesPreviousPaths[ptr.Val(s.GetId())] = sp.String()
|
|
|
|
cs, canUsePreviousBackup, err := site.CollectLibraries(
|
|
ictx,
|
|
sbpc,
|
|
bh,
|
|
creds.AzureTenantID,
|
|
ssmb,
|
|
su,
|
|
scl,
|
|
errs)
|
|
if err != nil {
|
|
el.AddRecoverable(ictx, err)
|
|
continue
|
|
}
|
|
|
|
if !canUsePreviousBackup {
|
|
dbcs = append(dbcs, data.NewTombstoneCollection(sp, control.Options{}, scl))
|
|
}
|
|
|
|
dbcs = append(dbcs, cs...)
|
|
}
|
|
case path.ChannelMessagesCategory:
|
|
var (
|
|
cs []data.BackupCollection
|
|
canUsePreviousBackup bool
|
|
err error
|
|
)
|
|
|
|
pcfg := observe.ProgressCfg{
|
|
Indent: 1,
|
|
// TODO(meain): Use number of messages and not channels
|
|
CompletionMessage: func() string { return fmt.Sprintf("(found %d channels)", len(cs)) },
|
|
}
|
|
progressBar := observe.MessageWithCompletion(ictx, pcfg, scope.Category().PathType().HumanString())
|
|
|
|
if !isTeam {
|
|
continue
|
|
}
|
|
|
|
bh := groups.NewChannelBackupHandler(bpc.ProtectedResource.ID(), ac.Channels())
|
|
|
|
cs, canUsePreviousBackup, err = groups.CreateCollections(
|
|
ictx,
|
|
bpc,
|
|
bh,
|
|
creds.AzureTenantID,
|
|
scope,
|
|
su,
|
|
cl,
|
|
errs)
|
|
if err != nil {
|
|
el.AddRecoverable(ictx, err)
|
|
continue
|
|
}
|
|
|
|
if !canUsePreviousBackup {
|
|
tp, err := bh.PathPrefix(creds.AzureTenantID)
|
|
if err != nil {
|
|
return nil, nil, clues.WrapWC(ictx, err, "getting message path").Label(count.BadPathPrefix)
|
|
}
|
|
|
|
dbcs = append(dbcs, data.NewTombstoneCollection(tp, control.Options{}, cl))
|
|
}
|
|
|
|
dbcs = append(dbcs, cs...)
|
|
|
|
close(progressBar)
|
|
}
|
|
|
|
collections = append(collections, dbcs...)
|
|
|
|
categories[scope.Category().PathType()] = struct{}{}
|
|
}
|
|
|
|
if len(collections) > 0 {
|
|
baseCols, err := graph.BaseCollections(
|
|
ctx,
|
|
collections,
|
|
creds.AzureTenantID,
|
|
bpc.ProtectedResource.ID(),
|
|
path.GroupsService,
|
|
categories,
|
|
su,
|
|
counter,
|
|
errs)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
collections = append(collections, baseCols...)
|
|
}
|
|
|
|
// Add metadata about sites
|
|
md, err := getSitesMetadataCollection(
|
|
creds.AzureTenantID,
|
|
bpc.ProtectedResource.ID(),
|
|
sitesPreviousPaths,
|
|
su,
|
|
counter)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
collections = append(collections, md)
|
|
|
|
counter.Add(count.Collections, int64(len(collections)))
|
|
|
|
logger.Ctx(ctx).Infow("produced collections", "stats", counter.Values())
|
|
|
|
return collections, ssmb.ToReader(), el.Failure()
|
|
}
|
|
|
|
func getSitesMetadataCollection(
|
|
tenantID, groupID string,
|
|
sites map[string]string,
|
|
su support.StatusUpdater,
|
|
counter *count.Bus,
|
|
) (data.BackupCollection, error) {
|
|
p, err := path.BuildMetadata(
|
|
tenantID,
|
|
groupID,
|
|
path.GroupsService,
|
|
path.LibrariesCategory,
|
|
false)
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "making metadata path")
|
|
}
|
|
|
|
p, err = p.Append(false, odConsts.SitesPathDir)
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "appending sites to metadata path")
|
|
}
|
|
|
|
md, err := graph.MakeMetadataCollection(
|
|
p,
|
|
[]graph.MetadataCollectionEntry{
|
|
graph.NewMetadataEntry(metadata.PreviousPathFileName, sites),
|
|
},
|
|
su,
|
|
counter.Local())
|
|
|
|
return md, err
|
|
}
|
|
|
|
func MetadataFiles(
|
|
ctx context.Context,
|
|
reason identity.Reasoner,
|
|
r kinject.RestoreProducer,
|
|
manID manifest.ID,
|
|
errs *fault.Bus,
|
|
) ([][]string, error) {
|
|
pth, err := path.BuildMetadata(
|
|
reason.Tenant(),
|
|
reason.ProtectedResource(),
|
|
reason.Service(),
|
|
reason.Category(),
|
|
true,
|
|
odConsts.SitesPathDir,
|
|
metadata.PreviousPathFileName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
dir, err := pth.Dir()
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "building metadata collection path")
|
|
}
|
|
|
|
dcs, err := r.ProduceRestoreCollections(
|
|
ctx,
|
|
string(manID),
|
|
[]path.RestorePaths{{StoragePath: pth, RestorePath: dir}},
|
|
nil,
|
|
errs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sites, err := deserializeSiteMetadata(ctx, dcs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
filePaths := [][]string{}
|
|
|
|
for k := range sites {
|
|
for _, fn := range metadata.AllMetadataFileNames() {
|
|
filePaths = append(filePaths, []string{odConsts.SitesPathDir, k, fn})
|
|
}
|
|
}
|
|
|
|
return filePaths, nil
|
|
}
|
|
|
|
func deserializeSiteMetadata(
|
|
ctx context.Context,
|
|
cols []data.RestoreCollection,
|
|
) (map[string]string, error) {
|
|
logger.Ctx(ctx).Infow(
|
|
"deserializing previous sites metadata",
|
|
"num_collections", len(cols))
|
|
|
|
var (
|
|
prevFolders = map[string]string{}
|
|
errs = fault.New(true) // metadata item reads should not fail backup
|
|
)
|
|
|
|
for _, col := range cols {
|
|
if errs.Failure() != nil {
|
|
break
|
|
}
|
|
|
|
items := col.Items(ctx, errs)
|
|
|
|
for breakLoop := false; !breakLoop; {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, clues.WrapWC(ctx, ctx.Err(), "deserializing previous sites metadata")
|
|
|
|
case item, ok := <-items:
|
|
if !ok {
|
|
breakLoop = true
|
|
break
|
|
}
|
|
|
|
var (
|
|
err error
|
|
ictx = clues.Add(ctx, "item_uuid", item.ID())
|
|
)
|
|
|
|
switch item.ID() {
|
|
case metadata.PreviousPathFileName:
|
|
err = drive.DeserializeMap(item.ToReader(), prevFolders)
|
|
|
|
default:
|
|
logger.Ctx(ictx).Infow(
|
|
"skipping unknown metadata file",
|
|
"file_name", item.ID())
|
|
|
|
continue
|
|
}
|
|
|
|
if err == nil {
|
|
// Successful decode.
|
|
continue
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, clues.StackWC(ictx, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// if reads from items failed, return empty but no error
|
|
if errs.Failure() != nil {
|
|
logger.CtxErr(ctx, errs.Failure()).Info("reading metadata collection items")
|
|
|
|
return map[string]string{}, nil
|
|
}
|
|
|
|
return prevFolders, nil
|
|
}
|