corso/src/internal/m365/collection/groups/backup.go

package groups

import (
	"context"

	"github.com/alcionai/clues"
	"github.com/microsoftgraph/msgraph-sdk-go/models"
	"golang.org/x/exp/maps"

	"github.com/alcionai/corso/src/internal/common/pii"
	"github.com/alcionai/corso/src/internal/common/ptr"
	"github.com/alcionai/corso/src/internal/common/str"
	"github.com/alcionai/corso/src/internal/data"
	"github.com/alcionai/corso/src/internal/m365/graph"
	"github.com/alcionai/corso/src/internal/m365/support"
	"github.com/alcionai/corso/src/internal/operations/inject"
	"github.com/alcionai/corso/src/pkg/backup/metadata"
	"github.com/alcionai/corso/src/pkg/control"
	"github.com/alcionai/corso/src/pkg/fault"
	"github.com/alcionai/corso/src/pkg/logger"
	"github.com/alcionai/corso/src/pkg/path"
	"github.com/alcionai/corso/src/pkg/selectors"
	"github.com/alcionai/corso/src/pkg/services/m365/api"
)

// TODO: incremental support
// multiple lines in this file are commented out so that
// we can focus on v0 backups and re-integrate them later
// for v1 incrementals.
// since these lines represent otherwise standard boilerplate,
// it's simpler to comment them for tracking than to delete
// and re-discover them later.

func CreateCollections(
	ctx context.Context,
	bpc inject.BackupProducerConfig,
	bh backupHandler,
	tenantID string,
	scope selectors.GroupsScope,
	su support.StatusUpdater,
	errs *fault.Bus,
) ([]data.BackupCollection, bool, error) {
	ctx = clues.Add(ctx, "category", scope.Category().PathType())

	var (
		allCollections = make([]data.BackupCollection, 0)
		category       = scope.Category().PathType()
		qp             = graph.QueryParams{
			Category:          category,
			ProtectedResource: bpc.ProtectedResource,
			TenantID:          tenantID,
		}
	)

	cdps, canUsePreviousBackup, err := parseMetadataCollections(ctx, bpc.MetadataCollections)
	if err != nil {
		return nil, false, err
	}

	ctx = clues.Add(ctx, "can_use_previous_backup", canUsePreviousBackup)

	channels, err := bh.getContainers(ctx)
	if err != nil {
		return nil, false, clues.Stack(err)
	}

	collections, err := populateCollections(
		ctx,
		qp,
		bh,
		su,
		channels,
		scope,
		cdps[scope.Category().PathType()],
		bpc.Options,
		errs)
	if err != nil {
		return nil, false, clues.Wrap(err, "filling collections")
	}

	for _, coll := range collections {
		allCollections = append(allCollections, coll)
	}

	return allCollections, canUsePreviousBackup, nil
}

func populateCollections(
	ctx context.Context,
	qp graph.QueryParams,
	bh backupHandler,
	statusUpdater support.StatusUpdater,
	channels []models.Channelable,
	scope selectors.GroupsScope,
	dps metadata.DeltaPaths,
	ctrlOpts control.Options,
	errs *fault.Bus,
) (map[string]data.BackupCollection, error) {
	var (
		// channel ID -> BackupCollection.
		collections = map[string]data.BackupCollection{}
		// channel ID -> delta url or folder path lookups
		deltaURLs = map[string]string{}
		currPaths = map[string]string{}
		// copy of previousPaths.  every channel present in the slice param
		// gets removed from this map; the remaining channels at the end of
		// the process have been deleted.
		tombstones = makeTombstones(dps)
		el         = errs.Local()
	)

	logger.Ctx(ctx).Info("filling collections", "len_deltapaths", len(dps))

	for _, c := range channels {
		if el.Failure() != nil {
			return nil, el.Failure()
		}

		var (
			cID         = ptr.Val(c.GetId())
			cName       = ptr.Val(c.GetDisplayName())
			err         error
			dp          = dps[cID]
			prevDelta   = dp.Delta
			prevPathStr = dp.Path // do not log: pii; log prevPath instead
			prevPath    path.Path
			ictx        = clues.Add(
				ctx,
				"channel_id", cID,
				"previous_delta", pii.SafeURL{
					URL:           prevDelta,
					SafePathElems: graph.SafeURLPathParams,
					SafeQueryKeys: graph.SafeURLQueryParams,
				})
		)

		delete(tombstones, cID)

		// Only create a collection if the path matches the scope.
		if !bh.includeContainer(ictx, qp, c, scope) {
			continue
		}

		if len(prevPathStr) > 0 {
			if prevPath, err = pathFromPrevString(prevPathStr); err != nil {
				logger.CtxErr(ictx, err).Error("parsing prev path")
				// if the previous path is unusable, then the delta must be, too.
				prevDelta = ""
			}
		}

		ictx = clues.Add(ictx, "previous_path", prevPath)

		// if the channel has no email property, it is unable to process delta tokens
		// and will return an error if a delta token is queried.
		cc := api.CallConfig{
			CanMakeDeltaQueries: len(ptr.Val(c.GetEmail())) > 0,
		}

		addAndRem, err := bh.getContainerItemIDs(ctx, cID, prevDelta, cc)
		if err != nil {
			el.AddRecoverable(ctx, clues.Stack(err))
			continue
		}

		added := str.SliceToMap(maps.Keys(addAndRem.Added))
		removed := str.SliceToMap(addAndRem.Removed)

		if len(addAndRem.DU.URL) > 0 {
			deltaURLs[cID] = addAndRem.DU.URL
		} else if !addAndRem.DU.Reset {
			logger.Ctx(ictx).Info("missing delta url")
		}

		currPath, err := bh.canonicalPath(path.Builder{}.Append(cID), qp.TenantID)
		if err != nil {
			el.AddRecoverable(ctx, clues.Stack(err))
			continue
		}

		// Remove any deleted IDs from the set of added IDs because items that are
		// deleted and then restored will have a different ID than they did
		// originally.
		for remove := range removed {
			delete(added, remove)
		}

		edc := NewCollection(
			data.NewBaseCollection(
				currPath,
				prevPath,
				path.Builder{}.Append(cName),
				ctrlOpts,
				addAndRem.DU.Reset),
			bh,
			qp.ProtectedResource.ID(),
			added,
			removed,
			statusUpdater)

		collections[cID] = &edc

		// add the current path for the container ID to be used in the next backup
		// as the "previous path", for reference in case of a rename or relocation.
		currPaths[cID] = currPath.String()
	}

	// A tombstone is a channel that needs to be marked for deletion.
	// The only situation where a tombstone should appear is if the channel exists
	// in the `previousPath` set, but does not exist in the enumeration.
	for id, p := range tombstones {
		if el.Failure() != nil {
			return nil, el.Failure()
		}

		var (
			err  error
			ictx = clues.Add(ctx, "tombstone_id", id)
		)

		if collections[id] != nil {
			el.AddRecoverable(ctx, clues.Wrap(err, "conflict: tombstone exists for a live collection").WithClues(ictx))
			continue
		}

		// only occurs if it was a new folder that we picked up during the container
		// resolver phase that got deleted in flight by the time we hit this stage.
		if len(p) == 0 {
			continue
		}

		prevPath, err := pathFromPrevString(p)
		if err != nil {
			// technically shouldn't ever happen.  But just in case...
			logger.CtxErr(ictx, err).Error("parsing tombstone prev path")
			continue
		}

		collections[id] = data.NewTombstoneCollection(prevPath, ctrlOpts)
	}

	logger.Ctx(ctx).Infow(
		"adding metadata collection entries",
		"num_deltas_entries", len(deltaURLs),
		"num_paths_entries", len(collections))

	pathPrefix, err := path.BuildMetadata(
		qp.TenantID,
		qp.ProtectedResource.ID(),
		path.GroupsService,
		qp.Category,
		false)
	if err != nil {
		return nil, clues.Wrap(err, "making metadata path prefix")
	}

	col, err := graph.MakeMetadataCollection(
		pathPrefix,
		[]graph.MetadataCollectionEntry{
			graph.NewMetadataEntry(metadata.PreviousPathFileName, currPaths),
			graph.NewMetadataEntry(metadata.DeltaURLsFileName, deltaURLs),
		},
		statusUpdater)
	if err != nil {
		return nil, clues.Wrap(err, "making metadata collection")
	}

	collections["metadata"] = col

	return collections, el.Failure()
}