Use previous snapshots to allow kopia to find previous versions of files (#1430)

## Description

Tell kopia about previous snapshots during the backup
so it can make use of them to skip uploading some data.

Currently only selects the most recent completed snapshot
and most recent incomplete snapshot that contains the
information being backed up. This can be tuned later if
it is not working good enough. However, increasing the
number of previous snapshots passed in may increase memory
usage during backup

## Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

* closes #1404

merge after:
* #1427 

## Test Plan

- [ ] 💪 Manual
- [x]  Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2022-12-02 08:25:16 -08:00 committed by GitHub
parent 31971bf356
commit 743c614ce4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 4 deletions

View File

@ -10,6 +10,7 @@ import (
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/pkg/errors"
@ -59,6 +60,8 @@ func IsRepoAlreadyExistsError(e error) bool {
return errors.As(e, &erae)
}
var _ snapshotManager = &conn{}
type conn struct {
storage storage.Storage
repo.Repository
@ -379,3 +382,10 @@ func checkCompressor(compressor compression.Name) error {
return errors.Errorf("unknown compressor type %s", compressor)
}
func (w *conn) LoadSnapshots(
ctx context.Context,
ids []manifest.ID,
) ([]*snapshot.Manifest, error) {
return snapshot.LoadSnapshots(ctx, w.Repository, ids)
}

View File

@ -187,9 +187,6 @@ func fetchPrevManifests(
// incomplete. An incomplete manifest may be returned if it is newer than the
// newest complete manifest for the tuple. Manifests are deduped such that if
// multiple tuples match the same manifest it will only be returned once.
//
// TODO(ashmrtn): Use to get previous manifests so backup can find previously
// uploaded versions of a file.
func fetchPrevSnapshotManifests(
ctx context.Context,
sm snapshotManager,

View File

@ -530,6 +530,8 @@ func (w Wrapper) makeSnapshotWithRoot(
) (*BackupStats, error) {
var man *snapshot.Manifest
prevSnaps := fetchPrevSnapshotManifests(ctx, w.c, oc)
bc := &stats.ByteCounter{}
err := repo.WriteSession(
@ -569,7 +571,7 @@ func (w Wrapper) makeSnapshotWithRoot(
progress.UploadProgress = u.Progress
u.Progress = progress
man, err = u.Upload(innerCtx, root, policyTree, si)
man, err = u.Upload(innerCtx, root, policyTree, si, prevSnaps...)
if err != nil {
err = errors.Wrap(err, "uploading data")
logger.Ctx(innerCtx).Errorw("kopia backup", err)