From 2b7d3877a4cfee51ec0b3fe0ad6237bfed1b5a5a Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Tue, 13 Sep 2022 09:29:17 -0700 Subject: [PATCH] Remove code to restore directory subtrees (#826) Not used and don't have the time to figure out the best way to bend the path struct API to get it to work nicely. --- src/internal/kopia/wrapper.go | 137 +---------------------- src/internal/kopia/wrapper_test.go | 171 ++--------------------------- 2 files changed, 8 insertions(+), 300 deletions(-) diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 0359d3d37..1fadff30e 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -446,27 +446,12 @@ func (w Wrapper) collectItems( ctx context.Context, snapshotID string, itemPath []string, - isDirectory bool, ) ([]data.Collection, error) { e, err := w.getEntry(ctx, snapshotID, itemPath) if err != nil { return nil, err } - // The paths passed below is the path up to (but not including) the - // file/directory passed. - if isDirectory { - dir, ok := e.(fs.Directory) - if !ok { - return nil, errors.New("requested object is not a directory") - } - - c, err := restoreSubtree(ctx, dir, itemPath[:len(itemPath)-1]) - // For some reason tests error out if the multierror is nil but we don't - // call ErrorOrNil. - return c, err.ErrorOrNil() - } - f, ok := e.(fs.File) if !ok { return nil, errors.New("requested object is not a file") @@ -492,7 +477,7 @@ func (w Wrapper) RestoreSingleItem( snapshotID string, itemPath []string, ) (data.Collection, error) { - c, err := w.collectItems(ctx, snapshotID, itemPath, false) + c, err := w.collectItems(ctx, snapshotID, itemPath) if err != nil { return nil, err } @@ -528,126 +513,6 @@ func restoreSingleItem( }, nil } -func walkDirectory( - ctx context.Context, - dir fs.Directory, -) ([]fs.File, []fs.Directory, *multierror.Error) { - var errs *multierror.Error - - files := []fs.File{} - dirs := []fs.Directory{} - - err := dir.IterateEntries(ctx, func(innerCtx context.Context, e fs.Entry) error { - // Early exit on context cancel. - if err := innerCtx.Err(); err != nil { - return err - } - - switch e := e.(type) { - case fs.Directory: - dirs = append(dirs, e) - case fs.File: - files = append(files, e) - default: - errs = multierror.Append(errs, errors.Errorf("unexpected item type %T", e)) - logger.Ctx(ctx).Errorw( - "unexpected item type; skipping", "type", e) - } - - return nil - }) - if err != nil { - // If the iterator itself had an error add it to the list. - errs = multierror.Append(errs, errors.Wrap(err, "getting directory data")) - } - - return files, dirs, errs -} - -// restoreSubtree returns DataCollections for each subdirectory (or the -// directory itself) that contains files. The FullPath of each returned -// DataCollection is the path from the root of the kopia directory structure to -// the directory. The UUID of each DataStream in each DataCollection is the name -// of the kopia file the data is sourced from. -func restoreSubtree( - ctx context.Context, - dir fs.Directory, - relativePath []string, -) ([]data.Collection, *multierror.Error) { - var errs *multierror.Error - - collections := []data.Collection{} - // Want a local copy of relativePath with our new element. - fullPath := append(append([]string{}, relativePath...), dir.Name()) - - files, dirs, err := walkDirectory(ctx, dir) - if err != nil { - errs = multierror.Append( - errs, errors.Wrapf(err, "walking directory %q", path.Join(fullPath...))) - } - - if len(files) > 0 { - if ctxErr := ctx.Err(); ctxErr != nil { - errs = multierror.Append(errs, errors.WithStack(ctxErr)) - return nil, errs - } - - streams := make([]data.Stream, 0, len(files)) - - for _, f := range files { - r, err := f.Open(ctx) - if err != nil { - fileFullPath := path.Join(append(append([]string{}, fullPath...), f.Name())...) - errs = multierror.Append( - errs, errors.Wrapf(err, "getting reader for file %q", fileFullPath)) - - logger.Ctx(ctx).Errorw( - "unable to get file reader; skipping", "path", fileFullPath) - - continue - } - - streams = append(streams, &kopiaDataStream{ - reader: r, - uuid: f.Name(), - }) - } - - collections = append(collections, &kopiaDataCollection{ - streams: streams, - path: fullPath, - }) - } - - for _, d := range dirs { - if ctxErr := ctx.Err(); ctxErr != nil { - errs = multierror.Append(errs, errors.WithStack(ctxErr)) - return nil, errs - } - - c, err := restoreSubtree(ctx, d, fullPath) - if err != nil { - errs = multierror.Append(errs, errors.Wrapf( - err, - "traversing subdirectory %q", - path.Join(append(append([]string{}, fullPath...), d.Name())...), - )) - } - - collections = append(collections, c...) - } - - return collections, errs -} - -func (w Wrapper) RestoreDirectory( - ctx context.Context, - snapshotID string, - basePath []string, -) ([]data.Collection, error) { - return w.collectItems(ctx, snapshotID, basePath, true) -} - // RestoreSingleItem looks up all paths- assuming each is an item declaration, // not a directory- in the snapshot with id snapshotID. The path should be the // full path of the item from the root. Returns the results as a slice of single- diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 6ba5c21b7..14efaa4b5 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -10,7 +10,6 @@ import ( "github.com/google/uuid" "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/stretchr/testify/assert" @@ -429,87 +428,6 @@ func (suite *KopiaUnitSuite) TestRestoreItem() { assert.Error(suite.T(), err) } -func (suite *KopiaUnitSuite) TestRestoreDirectory_FailGettingReader() { - ctx := context.Background() - t := suite.T() - - expectedStreamData := map[string][]byte{ - path.Join(testInboxDir, testFileName): testFileData, - path.Join(testInboxDir, testFileName3): testFileData3, - } - - dirs := virtualfs.NewStaticDirectory(testInboxDir, []fs.Entry{ - &mockkopia.MockFile{ - Entry: &mockkopia.MockEntry{ - EntryName: testFileName, - EntryMode: mockkopia.DefaultPermissions, - }, - Data: testFileData, - }, - &mockkopia.MockFile{ - Entry: &mockkopia.MockEntry{ - EntryName: testFileName2, - EntryMode: mockkopia.DefaultPermissions, - }, - OpenErr: assert.AnError, - }, - &mockkopia.MockFile{ - Entry: &mockkopia.MockEntry{ - EntryName: testFileName3, - EntryMode: mockkopia.DefaultPermissions, - }, - Data: testFileData3, - }, - }) - - collections, err := restoreSubtree(ctx, dirs, nil) - assert.Error(t, err) - - assert.Len(t, collections, 1) - testForFiles(t, expectedStreamData, collections) -} - -func (suite *KopiaUnitSuite) TestRestoreDirectory_FailWrongItemType() { - ctx := context.Background() - t := suite.T() - - expectedStreamData := map[string][]byte{ - path.Join(testEmailDir, testInboxDir, testFileName): testFileData, - path.Join(testEmailDir, testArchiveDir, testFileName3): testFileData3, - } - - dirs := virtualfs.NewStaticDirectory(testEmailDir, []fs.Entry{ - virtualfs.NewStaticDirectory(testInboxDir, []fs.Entry{ - &mockkopia.MockFile{ - Entry: &mockkopia.MockEntry{ - EntryName: testFileName, - EntryMode: mockkopia.DefaultPermissions, - }, - Data: testFileData, - }, - }), - virtualfs.NewStaticDirectory("foo", []fs.Entry{ - virtualfs.StreamingFileFromReader( - testFileName2, bytes.NewReader(testFileData2)), - }), - virtualfs.NewStaticDirectory(testArchiveDir, []fs.Entry{ - &mockkopia.MockFile{ - Entry: &mockkopia.MockEntry{ - EntryName: testFileName3, - EntryMode: mockkopia.DefaultPermissions, - }, - Data: testFileData3, - }, - }), - }) - - collections, err := restoreSubtree(ctx, dirs, nil) - assert.Error(t, err) - - assert.Len(t, collections, 2) - testForFiles(t, expectedStreamData, collections) -} - // --------------- // integration tests that use kopia // --------------- @@ -840,82 +758,6 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupAndRestoreSingleItem_Err } } -func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupRestoreDirectory() { - table := []struct { - name string - dirPath []string - expectedFiles map[string][]byte - }{ - { - "RecoverUser", - []string{testTenant, testUser}, - suite.allExpectedFiles, - }, - { - "RecoverMail", - []string{testTenant, testUser, testEmailDir}, - suite.allExpectedFiles, - }, - { - "RecoverInbox", - []string{testTenant, testUser, testEmailDir, testInboxDir}, - suite.inboxExpectedFiles, - }, - { - "RecoverArchive", - []string{testTenant, testUser, testEmailDir, testArchiveDir}, - suite.archiveExpectedFiles, - }, - } - - for _, test := range table { - suite.T().Run(test.name, func(t *testing.T) { - collections, err := suite.w.RestoreDirectory( - suite.ctx, string(suite.snapshotID), test.dirPath) - require.NoError(t, err) - - testForFiles(t, test.expectedFiles, collections) - }) - } -} - -func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupRestoreDirectory_Errors() { - table := []struct { - name string - snapshotID string - dirPath []string - }{ - { - "EmptyPath", - string(suite.snapshotID), - []string{}, - }, - { - "BadSnapshotID", - "foo", - []string{testTenant, testUser, testEmailDir}, - }, - { - "NotADirectory", - string(suite.snapshotID), - append(testPath, testFileName), - }, - { - "NonExistantDirectory", - string(suite.snapshotID), - []string{testTenant, testUser, testEmailDir, "subdir"}, - }, - } - - for _, test := range table { - suite.T().Run(test.name, func(t *testing.T) { - _, err := suite.w.RestoreDirectory( - suite.ctx, test.snapshotID, test.dirPath) - assert.Error(t, err) - }) - } -} - func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() { t := suite.T() ctx := context.Background() @@ -996,11 +838,12 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors() func (suite *KopiaIntegrationSuite) TestDeleteSnapshot() { t := suite.T() + dc1 := mockconnector.NewMockExchangeCollection( + []string{"a-tenant", "user1", "emails"}, + 5, + ) collections := []data.Collection{ - mockconnector.NewMockExchangeCollection( - []string{"a-tenant", "user1", "emails"}, - 5, - ), + dc1, mockconnector.NewMockExchangeCollection( []string{"a-tenant", "user2", "emails"}, 42, @@ -1014,8 +857,8 @@ func (suite *KopiaIntegrationSuite) TestDeleteSnapshot() { assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, snapshotID)) // assert the deletion worked - dirPath := []string{testTenant, testUser} - _, err = suite.w.RestoreDirectory(suite.ctx, snapshotID, dirPath) + itemPath := []string{"a-tenant", "user1", "emails", dc1.Names[0]} + _, err = suite.w.RestoreSingleItem(suite.ctx, snapshotID, itemPath) assert.Error(t, err, "snapshot should be deleted") }