corso/src/internal/operations/manifests.go

package operations

import (
	"context"

	"github.com/alcionai/clues"
	"github.com/pkg/errors"

	"github.com/alcionai/corso/src/internal/data"
	"github.com/alcionai/corso/src/internal/kopia"
	"github.com/alcionai/corso/src/internal/kopia/inject"
	"github.com/alcionai/corso/src/internal/m365/graph"
	"github.com/alcionai/corso/src/pkg/fault"
	"github.com/alcionai/corso/src/pkg/logger"
	"github.com/alcionai/corso/src/pkg/path"
)

// calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics.
// TODO(ashmrtn): Make this a helper function that always returns as much as
// possible and call in another function that drops metadata and/or
// kopia-assisted incremental bases based on flag values.
func produceManifestsAndMetadata(
	ctx context.Context,
	bf inject.BaseFinder,
	rp inject.RestoreProducer,
	reasons, fallbackReasons []kopia.Reason,
	tenantID string,
	getMetadata bool,
) (kopia.BackupBases, []data.RestoreCollection, bool, error) {
	var (
		tags          = map[string]string{kopia.TagBackupCategory: ""}
		metadataFiles = graph.AllMetadataFileNames()
		collections   []data.RestoreCollection
	)

	bb := bf.FindBases(ctx, reasons, tags)
	// TODO(ashmrtn): Only fetch these if we haven't already covered all the
	// reasons for this backup.
	fbb := bf.FindBases(ctx, fallbackReasons, tags)

	// one of three cases can occur when retrieving backups across reason migrations:
	// 1. the current reasons don't match any manifests, and we use the fallback to
	// look up the previous reason version.
	// 2. the current reasons only contain an incomplete manifest, and the fallback
	// can find a complete manifest.
	// 3. the current reasons contain all the necessary manifests.
	bb = bb.MergeBackupBases(
		ctx,
		fbb,
		func(r kopia.Reason) string {
			return r.Service.String() + r.Category.String()
		})

	if !getMetadata {
		logger.Ctx(ctx).Debug("full backup requested, dropping merge bases")

		// TODO(ashmrtn): If this function is moved to be a helper function then
		// move this change to the bases to the caller of this function.
		bb.ClearMergeBases()

		return bb, nil, false, nil
	}

	for _, man := range bb.MergeBases() {
		mctx := clues.Add(ctx, "manifest_id", man.ID)

		// a local fault.Bus intance is used to collect metadata files here.
		// we avoid the global fault.Bus because all failures here are ignorable,
		// and cascading errors up to the operation can cause a conflict that forces
		// the operation into a failure state unnecessarily.
		// TODO(keepers): this is not a pattern we want to
		// spread around.  Need to find more idiomatic handling.
		fb := fault.New(true)

		colls, err := collectMetadata(mctx, rp, man, metadataFiles, tenantID, fb)
		LogFaultErrors(ctx, fb.Errors(), "collecting metadata")

		// TODO(ashmrtn): It should be alright to relax this condition a little. We
		// should be able to just remove the offending manifest and backup from the
		// set of bases. Since we're looking at manifests in this loop, it should be
		// possible to find the backup by either checking the reasons or extracting
		// the backup ID from the manifests tags.
		//
		// Assuming that only the corso metadata is corrupted for the manifest, it
		// should be safe to leave this manifest in the AssistBases set, though we
		// could remove it there too if we want to be conservative. That can be done
		// by finding the manifest ID.
		if err != nil && !errors.Is(err, data.ErrNotFound) {
			// prior metadata isn't guaranteed to exist.
			// if it doesn't, we'll just have to do a
			// full backup for that data.
			return nil, nil, false, err
		}

		collections = append(collections, colls...)
	}

	return bb, collections, true, nil
}

// collectMetadata retrieves all metadata files associated with the manifest.
func collectMetadata(
	ctx context.Context,
	r inject.RestoreProducer,
	man kopia.ManifestEntry,
	fileNames []string,
	tenantID string,
	errs *fault.Bus,
) ([]data.RestoreCollection, error) {
	paths := []path.RestorePaths{}

	for _, fn := range fileNames {
		for _, reason := range man.Reasons {
			p, err := path.Builder{}.
				Append(fn).
				ToServiceCategoryMetadataPath(
					tenantID,
					reason.ResourceOwner,
					reason.Service,
					reason.Category,
					true)
			if err != nil {
				return nil, clues.
					Wrap(err, "building metadata path").
					With("metadata_file", fn, "category", reason.Category)
			}

			dir, err := p.Dir()
			if err != nil {
				return nil, clues.
					Wrap(err, "building metadata collection path").
					With("metadata_file", fn, "category", reason.Category)
			}

			paths = append(paths, path.RestorePaths{StoragePath: p, RestorePath: dir})
		}
	}

	dcs, err := r.ProduceRestoreCollections(ctx, string(man.ID), paths, nil, errs)
	if err != nil {
		// Restore is best-effort and we want to keep it that way since we want to
		// return as much metadata as we can to reduce the work we'll need to do.
		// Just wrap the error here for better reporting/debugging.
		return dcs, clues.Wrap(err, "collecting prior metadata")
	}

	return dcs, nil
}