corso/src/internal/operations/manifests.go

package operations

import (
	"context"

	"github.com/alcionai/clues"
	"github.com/pkg/errors"

	"github.com/alcionai/corso/src/internal/data"
	"github.com/alcionai/corso/src/internal/kopia"
	"github.com/alcionai/corso/src/internal/kopia/inject"
	oinject "github.com/alcionai/corso/src/internal/operations/inject"
	"github.com/alcionai/corso/src/pkg/backup/identity"
	"github.com/alcionai/corso/src/pkg/fault"
	"github.com/alcionai/corso/src/pkg/logger"
)

func produceManifestsAndMetadata(
	ctx context.Context,
	bf inject.BaseFinder,
	bp oinject.BackupProducer,
	rp inject.RestoreProducer,
	reasons, fallbackReasons []identity.Reasoner,
	tenantID string,
	getMetadata, dropAssistBases bool,
) (kopia.BackupBases, []data.RestoreCollection, bool, error) {
	bb, meta, useMergeBases, err := getManifestsAndMetadata(
		ctx,
		bf,
		bp,
		rp,
		reasons,
		fallbackReasons,
		tenantID,
		getMetadata)
	if err != nil {
		return nil, nil, false, clues.Stack(err)
	}

	if !useMergeBases || !getMetadata {
		logger.Ctx(ctx).Debug("full backup requested, dropping merge bases")

		bb.DisableMergeBases()
	}

	if dropAssistBases {
		logger.Ctx(ctx).Debug("no caching requested, dropping assist bases")

		bb.DisableAssistBases()
	}

	return bb, meta, useMergeBases, nil
}

// getManifestsAndMetadata calls kopia to retrieve prior backup manifests,
// metadata collections to supply backup information.
func getManifestsAndMetadata(
	ctx context.Context,
	bf inject.BaseFinder,
	bp oinject.BackupProducer,
	rp inject.RestoreProducer,
	reasons, fallbackReasons []identity.Reasoner,
	tenantID string,
	getMetadata bool,
) (kopia.BackupBases, []data.RestoreCollection, bool, error) {
	var (
		tags        = map[string]string{kopia.TagBackupCategory: ""}
		collections []data.RestoreCollection
	)

	bb := bf.FindBases(ctx, reasons, tags)
	// TODO(ashmrtn): Only fetch these if we haven't already covered all the
	// reasons for this backup.
	fbb := bf.FindBases(ctx, fallbackReasons, tags)

	// one of three cases can occur when retrieving backups across reason migrations:
	// 1. the current reasons don't match any manifests, and we use the fallback to
	// look up the previous reason version.
	// 2. the current reasons only contain an incomplete manifest, and the fallback
	// can find a complete manifest.
	// 3. the current reasons contain all the necessary manifests.
	// Note: This is not relevant for assist backups, since they are newly introduced
	// and they don't exist with fallback reasons.
	bb = bb.MergeBackupBases(
		ctx,
		fbb,
		func(r identity.Reasoner) string {
			return r.Service().String() + r.Category().String()
		})

	if !getMetadata {
		return bb, nil, false, nil
	}

	for _, base := range bb.MergeBases() {
		mctx := clues.Add(
			ctx,
			"base_item_data_snapshot_id", base.ItemDataSnapshot.ID,
			"base_backup_id", base.Backup.ID)

		// a local fault.Bus intance is used to collect metadata files here.
		// we avoid the global fault.Bus because all failures here are ignorable,
		// and cascading errors up to the operation can cause a conflict that forces
		// the operation into a failure state unnecessarily.
		// TODO(keepers): this is not a pattern we want to
		// spread around.  Need to find more idiomatic handling.
		fb := fault.New(true)

		paths, err := bp.GetMetadataPaths(mctx, rp, base, fb)
		if err != nil {
			LogFaultErrors(ctx, fb.Errors(), "collecting metadata paths")
			return nil, nil, false, err
		}

		colls, err := rp.ProduceRestoreCollections(
			ctx,
			string(base.ItemDataSnapshot.ID),
			paths,
			nil,
			fb)
		if err != nil {
			// Restore is best-effort and we want to keep it that way since we want to
			// return as much metadata as we can to reduce the work we'll need to do.
			// Just wrap the error here for better reporting/debugging.
			LogFaultErrors(ctx, fb.Errors(), "collecting metadata")
		}

		// TODO(ashmrtn): It should be alright to relax this condition a little. We
		// should be able to just remove the offending manifest and backup from the
		// set of bases. Since we're looking at manifests in this loop, it should be
		// possible to find the backup by either checking the reasons or extracting
		// the backup ID from the manifests tags.
		//
		// Assuming that only the corso metadata is corrupted for the manifest, it
		// should be safe to leave this manifest in the AssistBases set, though we
		// could remove it there too if we want to be conservative. That can be done
		// by finding the manifest ID.
		if err != nil && !errors.Is(err, data.ErrNotFound) {
			// prior metadata isn't guaranteed to exist.
			// if it doesn't, we'll just have to do a
			// full backup for that data.
			return nil, nil, false, err
		}

		collections = append(collections, colls...)
	}

	return bb, collections, true, nil
}