77c48ebf95
#### Does this PR need a docs update or release note? - [x] ✅ Yes, it's included #### Type of change - [x] 🐛 Bugfix #### Issue(s) * #3988 #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
342 lines
8.1 KiB
Go
342 lines
8.1 KiB
Go
package groups
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/alcionai/clues"
|
|
"github.com/kopia/kopia/repo/manifest"
|
|
|
|
"github.com/alcionai/corso/src/internal/common/idname"
|
|
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
|
"github.com/alcionai/corso/src/internal/data"
|
|
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/groups"
|
|
"github.com/alcionai/corso/src/internal/m365/collection/site"
|
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
|
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
|
|
"github.com/alcionai/corso/src/internal/m365/support"
|
|
"github.com/alcionai/corso/src/internal/observe"
|
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
|
"github.com/alcionai/corso/src/pkg/account"
|
|
"github.com/alcionai/corso/src/pkg/backup/identity"
|
|
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
|
"github.com/alcionai/corso/src/pkg/fault"
|
|
"github.com/alcionai/corso/src/pkg/logger"
|
|
"github.com/alcionai/corso/src/pkg/path"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
|
)
|
|
|
|
func ProduceBackupCollections(
|
|
ctx context.Context,
|
|
bpc inject.BackupProducerConfig,
|
|
ac api.Client,
|
|
creds account.M365Config,
|
|
su support.StatusUpdater,
|
|
errs *fault.Bus,
|
|
) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) {
|
|
b, err := bpc.Selector.ToGroupsBackup()
|
|
if err != nil {
|
|
return nil, nil, false, clues.Wrap(err, "groupsDataCollection: parsing selector")
|
|
}
|
|
|
|
var (
|
|
el = errs.Local()
|
|
collections = []data.BackupCollection{}
|
|
categories = map[path.CategoryType]struct{}{}
|
|
ssmb = prefixmatcher.NewStringSetBuilder()
|
|
canUsePreviousBackup bool
|
|
sitesPreviousPaths = map[string]string{}
|
|
)
|
|
|
|
ctx = clues.Add(
|
|
ctx,
|
|
"group_id", clues.Hide(bpc.ProtectedResource.ID()),
|
|
"group_name", clues.Hide(bpc.ProtectedResource.Name()))
|
|
|
|
group, err := ac.Groups().GetByID(ctx, bpc.ProtectedResource.ID())
|
|
if err != nil {
|
|
return nil, nil, false, clues.Wrap(err, "getting group").WithClues(ctx)
|
|
}
|
|
|
|
isTeam := api.IsTeam(ctx, group)
|
|
|
|
for _, scope := range b.Scopes() {
|
|
if el.Failure() != nil {
|
|
break
|
|
}
|
|
|
|
catStr := scope.Category().PathType().String()
|
|
if scope.Category().PathType() == path.ChannelMessagesCategory {
|
|
catStr = "messages"
|
|
}
|
|
|
|
progressBar := observe.MessageWithCompletion(
|
|
ctx,
|
|
observe.Bulletf("%s", catStr))
|
|
defer close(progressBar)
|
|
|
|
var dbcs []data.BackupCollection
|
|
|
|
switch scope.Category().PathType() {
|
|
case path.LibrariesCategory:
|
|
// TODO(meain): Private channels get a separate SharePoint
|
|
// site. We should also back those up and not just the
|
|
// default one.
|
|
resp, err := ac.Groups().GetRootSite(ctx, bpc.ProtectedResource.ID())
|
|
if err != nil {
|
|
return nil, nil, false, err
|
|
}
|
|
|
|
siteMetadataCollection := map[string][]data.RestoreCollection{}
|
|
|
|
// Once we have metadata collections for chat as well, we will have to filter those out
|
|
for _, c := range bpc.MetadataCollections {
|
|
siteID := c.FullPath().Elements().Last()
|
|
siteMetadataCollection[siteID] = append(siteMetadataCollection[siteID], c)
|
|
}
|
|
|
|
pr := idname.NewProvider(ptr.Val(resp.GetId()), ptr.Val(resp.GetName()))
|
|
sbpc := inject.BackupProducerConfig{
|
|
LastBackupVersion: bpc.LastBackupVersion,
|
|
Options: bpc.Options,
|
|
ProtectedResource: pr,
|
|
Selector: bpc.Selector,
|
|
MetadataCollections: siteMetadataCollection[ptr.Val(resp.GetId())],
|
|
}
|
|
|
|
bh := drive.NewGroupBackupHandler(
|
|
bpc.ProtectedResource.ID(),
|
|
ptr.Val(resp.GetId()),
|
|
ac.Drives(),
|
|
scope)
|
|
|
|
cp, err := bh.SitePathPrefix(creds.AzureTenantID)
|
|
if err != nil {
|
|
return nil, nil, false, clues.Wrap(err, "getting canonical path")
|
|
}
|
|
|
|
sitesPreviousPaths[ptr.Val(resp.GetId())] = cp.String()
|
|
|
|
dbcs, canUsePreviousBackup, err = site.CollectLibraries(
|
|
ctx,
|
|
sbpc,
|
|
bh,
|
|
creds.AzureTenantID,
|
|
ssmb,
|
|
su,
|
|
errs)
|
|
if err != nil {
|
|
el.AddRecoverable(ctx, err)
|
|
continue
|
|
}
|
|
|
|
case path.ChannelMessagesCategory:
|
|
if !isTeam {
|
|
continue
|
|
}
|
|
|
|
dbcs, canUsePreviousBackup, err = groups.CreateCollections(
|
|
ctx,
|
|
bpc,
|
|
groups.NewChannelBackupHandler(bpc.ProtectedResource.ID(), ac.Channels()),
|
|
creds.AzureTenantID,
|
|
scope,
|
|
su,
|
|
errs)
|
|
if err != nil {
|
|
el.AddRecoverable(ctx, err)
|
|
continue
|
|
}
|
|
}
|
|
|
|
collections = append(collections, dbcs...)
|
|
|
|
categories[scope.Category().PathType()] = struct{}{}
|
|
}
|
|
|
|
if len(collections) > 0 {
|
|
baseCols, err := graph.BaseCollections(
|
|
ctx,
|
|
collections,
|
|
creds.AzureTenantID,
|
|
bpc.ProtectedResource.ID(),
|
|
path.GroupsService,
|
|
categories,
|
|
su,
|
|
errs)
|
|
if err != nil {
|
|
return nil, nil, false, err
|
|
}
|
|
|
|
collections = append(collections, baseCols...)
|
|
}
|
|
|
|
// Add metadata about sites
|
|
md, err := getSitesMetadataCollection(
|
|
creds.AzureTenantID,
|
|
bpc.ProtectedResource.ID(),
|
|
sitesPreviousPaths,
|
|
su)
|
|
if err != nil {
|
|
return nil, nil, false, err
|
|
}
|
|
|
|
collections = append(collections, md)
|
|
|
|
return collections, ssmb.ToReader(), canUsePreviousBackup, el.Failure()
|
|
}
|
|
|
|
func getSitesMetadataCollection(
|
|
tenantID, groupID string,
|
|
sites map[string]string,
|
|
su support.StatusUpdater,
|
|
) (data.BackupCollection, error) {
|
|
p, err := path.BuildMetadata(
|
|
tenantID,
|
|
groupID,
|
|
path.GroupsService,
|
|
path.LibrariesCategory,
|
|
false)
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "making metadata path")
|
|
}
|
|
|
|
p, err = p.Append(false, odConsts.SitesPathDir)
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "appending sites to metadata path")
|
|
}
|
|
|
|
md, err := graph.MakeMetadataCollection(
|
|
p,
|
|
[]graph.MetadataCollectionEntry{
|
|
graph.NewMetadataEntry(metadata.PreviousPathFileName, sites),
|
|
},
|
|
su)
|
|
|
|
return md, err
|
|
}
|
|
|
|
func MetadataFiles(
|
|
ctx context.Context,
|
|
reason identity.Reasoner,
|
|
r kinject.RestoreProducer,
|
|
manID manifest.ID,
|
|
errs *fault.Bus,
|
|
) ([][]string, error) {
|
|
pth, err := path.BuildMetadata(
|
|
reason.Tenant(),
|
|
reason.ProtectedResource(),
|
|
reason.Service(),
|
|
reason.Category(),
|
|
true,
|
|
odConsts.SitesPathDir,
|
|
metadata.PreviousPathFileName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
dir, err := pth.Dir()
|
|
if err != nil {
|
|
return nil, clues.Wrap(err, "building metadata collection path")
|
|
}
|
|
|
|
dcs, err := r.ProduceRestoreCollections(
|
|
ctx,
|
|
string(manID),
|
|
[]path.RestorePaths{{StoragePath: pth, RestorePath: dir}},
|
|
nil,
|
|
errs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sites, err := deserializeSiteMetadata(ctx, dcs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
filePaths := [][]string{}
|
|
|
|
for k := range sites {
|
|
for _, fn := range metadata.AllMetadataFileNames() {
|
|
filePaths = append(filePaths, []string{odConsts.SitesPathDir, k, fn})
|
|
}
|
|
}
|
|
|
|
return filePaths, nil
|
|
}
|
|
|
|
func deserializeSiteMetadata(
|
|
ctx context.Context,
|
|
cols []data.RestoreCollection,
|
|
) (map[string]string, error) {
|
|
logger.Ctx(ctx).Infow(
|
|
"deserializing previous sites metadata",
|
|
"num_collections", len(cols))
|
|
|
|
var (
|
|
prevFolders = map[string]string{}
|
|
errs = fault.New(true) // metadata item reads should not fail backup
|
|
)
|
|
|
|
for _, col := range cols {
|
|
if errs.Failure() != nil {
|
|
break
|
|
}
|
|
|
|
items := col.Items(ctx, errs)
|
|
|
|
for breakLoop := false; !breakLoop; {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, clues.Wrap(
|
|
ctx.Err(),
|
|
"deserializing previous sites metadata").WithClues(ctx)
|
|
|
|
case item, ok := <-items:
|
|
if !ok {
|
|
breakLoop = true
|
|
break
|
|
}
|
|
|
|
var (
|
|
err error
|
|
ictx = clues.Add(ctx, "item_uuid", item.ID())
|
|
)
|
|
|
|
switch item.ID() {
|
|
case metadata.PreviousPathFileName:
|
|
err = drive.DeserializeMap(item.ToReader(), prevFolders)
|
|
|
|
default:
|
|
logger.Ctx(ictx).Infow(
|
|
"skipping unknown metadata file",
|
|
"file_name", item.ID())
|
|
|
|
continue
|
|
}
|
|
|
|
if err == nil {
|
|
// Successful decode.
|
|
continue
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, clues.Stack(err).WithClues(ictx)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// if reads from items failed, return empty but no error
|
|
if errs.Failure() != nil {
|
|
logger.CtxErr(ctx, errs.Failure()).Info("reading metadata collection items")
|
|
|
|
return map[string]string{}, nil
|
|
}
|
|
|
|
return prevFolders, nil
|
|
}
|