From 5ae8259fd5ee15c1a5260509d2f6a7ea9e003403 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 14 Sep 2022 10:01:11 -0700 Subject: [PATCH] Change return value of data.Collection.FullPath to path.Path (#841) * Make data.Collection.FullPath return path.Path * Fixup graph connector code for path struct * Add Elements call to Path interface Still should not be used in place of the named functions to get specific path elements (e.x. ResourceOwner()) * Fixup kopia wrapper path handling Mostly removing shim code and minor fixup for building the directory tree. * All the test fixes --- .../exchange/exchange_data_collection.go | 9 +- .../exchange/exchange_data_collection_test.go | 16 +- .../connector/exchange/iterators_test.go | 5 +- src/internal/connector/graph_connector.go | 23 +- .../graph_connector_disconnected_test.go | 10 - .../connector/graph_connector_test.go | 13 +- .../mockconnector/mock_data_collection.go | 9 +- .../mock_data_collection_test.go | 4 +- src/internal/connector/onedrive/collection.go | 7 +- .../connector/onedrive/collection_test.go | 7 +- src/internal/data/data_collection.go | 7 +- src/internal/kopia/data_collection.go | 7 +- src/internal/kopia/data_collection_test.go | 13 +- src/internal/kopia/wrapper.go | 21 +- src/internal/kopia/wrapper_test.go | 285 ++++++++++-------- src/internal/path/path.go | 4 + 16 files changed, 198 insertions(+), 242 deletions(-) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index ddccf7242..cfb9c4b31 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "io" - "strings" absser "github.com/microsoft/kiota-abstractions-go/serialization" kw "github.com/microsoft/kiota-serialization-json-go" @@ -108,12 +107,8 @@ func GetQueryAndSerializeFunc(optID optionIdentifier) (GraphRetrievalFunc, Graph } // FullPath returns the Collection's fullPath []string -func (col *Collection) FullPath() []string { - // TODO(ashmrtn): Remove this when data.Collection.FullPath returns a - // path.Path. This assumes we don't have adversarial users that use '/' in - // their folder names. - r := col.fullPath.String() - return strings.Split(r, "/") +func (col *Collection) FullPath() path.Path { + return col.fullPath } // populateByOptionIdentifier is a utility function that uses col.collectionType to be able to serialize diff --git a/src/internal/connector/exchange/exchange_data_collection_test.go b/src/internal/connector/exchange/exchange_data_collection_test.go index a6c61eb37..261b916e5 100644 --- a/src/internal/connector/exchange/exchange_data_collection_test.go +++ b/src/internal/connector/exchange/exchange_data_collection_test.go @@ -51,13 +51,6 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() { tenant := "a-tenant" user := "a-user" folder := "a-folder" - expected := []string{ - tenant, - path.ExchangeService.String(), - user, - path.EmailCategory.String(), - folder, - } fullPath, err := path.Builder{}.Append(folder).ToDataLayerExchangePathForCategory( tenant, @@ -72,8 +65,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() { fullPath: fullPath, } - // TODO(ashmrtn): Update when data.Collection.FullPath returns path.Path. - assert.Equal(t, expected, edc.FullPath()) + assert.Equal(t, fullPath, edc.FullPath()) } func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchangeDataCollection() { @@ -95,11 +87,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchange fullPath: fullPath, } suite.Equal(name, edc.user) - suite.Contains(edc.FullPath(), fullPath.Tenant()) - suite.Contains(edc.FullPath(), fullPath.Service().String()) - suite.Contains(edc.FullPath(), fullPath.Category().String()) - suite.Contains(edc.FullPath(), fullPath.ResourceOwner()) - suite.Contains(edc.FullPath(), fullPath.Folder()) + suite.Equal(fullPath, edc.FullPath()) } func (suite *ExchangeDataCollectionSuite) TestExchangeCollection_AddJob() { diff --git a/src/internal/connector/exchange/iterators_test.go b/src/internal/connector/exchange/iterators_test.go index 2a0279a35..fc34fe55f 100644 --- a/src/internal/connector/exchange/iterators_test.go +++ b/src/internal/connector/exchange/iterators_test.go @@ -190,10 +190,9 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() { } for _, c := range collections { - // TODO(ashmrtn): Update these checks when collections support path.Path. - require.Greater(t, len(c.FullPath()), 4) + require.NotEmpty(t, c.FullPath().Folder()) - folder := c.FullPath()[4] + folder := c.FullPath().Folder() if _, ok := test.folderNames[folder]; ok { delete(test.folderNames, folder) } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index cb7437bc4..a1eb2f492 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -6,7 +6,6 @@ import ( "bytes" "context" "fmt" - stdpath "path" "sync" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" @@ -259,24 +258,14 @@ func (gc *GraphConnector) RestoreDataCollections( for _, dc := range dcs { var ( - items = dc.Items() - exit bool + items = dc.Items() + directory = dc.FullPath() + service = directory.Service() + category = directory.Category() + user = directory.ResourceOwner() + exit bool ) - // TODO(ashmrtn): Remove this when data.Collection.FullPath supports path.Path - directory, err := path.FromDataLayerPath( - stdpath.Join(dc.FullPath()...), - false, - ) - if err != nil { - errs = support.WrapAndAppend("parsing Collection path", err, errs) - continue - } - - category := directory.Category() - user := directory.ResourceOwner() - service := directory.Service() - // Check whether restoring data into the specified service is supported switch service { case path.ExchangeService: diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 9205198f4..1853cb720 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -10,9 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/support" - "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/credentials" @@ -86,14 +84,6 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() { suite.Contains(last, "Foley") } -func (suite *DisconnectedGraphConnectorSuite) TestInterfaceAlignment() { - var dc data.Collection - - concrete := mockconnector.NewMockExchangeCollection([]string{"a", "path"}, 1) - dc = concrete - assert.NotNil(suite.T(), dc) -} - func statusTestTask(gc *GraphConnector, objects, success, folder int) { status := support.CreateStatus( context.Background(), diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 936b5ccf3..b977e27e9 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "strings" "testing" "time" @@ -99,7 +98,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { // Verify Items() call returns an iterable channel(e.g. a channel that has been closed) for _, collection := range collectionList { temp := collection.Items() - testName := collection.FullPath()[2] + testName := collection.FullPath().ResourceOwner() streams[testName] = temp } @@ -116,9 +115,6 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { } }) } - - exchangeData := collectionList[0] - suite.Greater(len(exchangeData.FullPath()), 2) } // TestMailSerializationRegression verifies that all mail data stored in the @@ -139,8 +135,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() { require.NoError(t, err) for _, edc := range collection { - testName := strings.Join(edc.FullPath(), " ") - suite.T().Run(testName, func(t *testing.T) { + suite.T().Run(edc.FullPath().String(), func(t *testing.T) { streamChannel := edc.Items() // Verify that each message can be restored for stream := range streamChannel { @@ -181,7 +176,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression( number := 0 for _, edc := range collections { - testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath()[1], number) + testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath().ResourceOwner(), number) suite.T().Run(testName, func(t *testing.T) { streamChannel := edc.Items() for stream := range streamChannel { @@ -220,7 +215,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() number := 0 for stream := range streamChannel { - testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[2], number) + testName := fmt.Sprintf("%s_Event_%d", edc.FullPath().ResourceOwner(), number) suite.T().Run(testName, func(t *testing.T) { buf := &bytes.Buffer{} read, err := buf.ReadFrom(stream.ToReader()) diff --git a/src/internal/connector/mockconnector/mock_data_collection.go b/src/internal/connector/mockconnector/mock_data_collection.go index b881644b2..980bc7705 100644 --- a/src/internal/connector/mockconnector/mock_data_collection.go +++ b/src/internal/connector/mockconnector/mock_data_collection.go @@ -10,12 +10,13 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/path" "github.com/alcionai/corso/src/pkg/backup/details" ) // MockExchangeDataCollection represents a mock exchange mailbox type MockExchangeDataCollection struct { - fullPath []string + fullPath path.Path messageCount int Data [][]byte Names []string @@ -29,7 +30,7 @@ var ( // NewMockExchangeDataCollection creates an data collection that will return the specified number of // mock messages when iterated. Exchange type mail -func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn int) *MockExchangeDataCollection { +func NewMockExchangeCollection(pathRepresentation path.Path, numMessagesToReturn int) *MockExchangeDataCollection { c := &MockExchangeDataCollection{ fullPath: pathRepresentation, messageCount: numMessagesToReturn, @@ -46,8 +47,8 @@ func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn return c } -func (medc *MockExchangeDataCollection) FullPath() []string { - return append([]string{}, medc.fullPath...) +func (medc *MockExchangeDataCollection) FullPath() path.Path { + return medc.fullPath } // Items returns a channel that has the next items in the collection. The diff --git a/src/internal/connector/mockconnector/mock_data_collection_test.go b/src/internal/connector/mockconnector/mock_data_collection_test.go index f18dd856c..c13990432 100644 --- a/src/internal/connector/mockconnector/mock_data_collection_test.go +++ b/src/internal/connector/mockconnector/mock_data_collection_test.go @@ -24,7 +24,7 @@ func TestMockExchangeCollectionSuite(t *testing.T) { } func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() { - mdc := mockconnector.NewMockExchangeCollection([]string{"foo", "bar"}, 2) + mdc := mockconnector.NewMockExchangeCollection(nil, 2) messagesRead := 0 @@ -41,7 +41,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() { // functions by verifying no failures on (de)serializing steps using kiota serialization library func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection_NewExchangeCollectionMail_Hydration() { t := suite.T() - mdc := mockconnector.NewMockExchangeCollection([]string{"foo", "bar"}, 3) + mdc := mockconnector.NewMockExchangeCollection(nil, 3) buf := &bytes.Buffer{} for stream := range mdc.Items() { diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index fa9b8871e..6799feb34 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -4,7 +4,6 @@ package onedrive import ( "context" "io" - "strings" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" @@ -80,10 +79,8 @@ func (oc *Collection) Items() <-chan data.Stream { return oc.data } -func (oc *Collection) FullPath() []string { - // TODO(ashmrtn): Update this when data.Collection.FullPath has support for - // path.Path. - return strings.Split(oc.folderPath.String(), "/") +func (oc *Collection) FullPath() path.Path { + return oc.folderPath } // Item represents a single item retrieved from OneDrive diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index 8c9b2c55a..add89f295 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "strings" "sync" "testing" @@ -64,11 +63,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() { coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus)) require.NotNil(t, coll) - assert.Equal( - t, - strings.Split(folderPath.String(), "/"), - coll.FullPath(), - ) + assert.Equal(t, folderPath, coll.FullPath()) testItemID := "fakeItemID" testItemName := "itemName" diff --git a/src/internal/data/data_collection.go b/src/internal/data/data_collection.go index c5cae4e3e..efd85e8a0 100644 --- a/src/internal/data/data_collection.go +++ b/src/internal/data/data_collection.go @@ -3,6 +3,7 @@ package data import ( "io" + "github.com/alcionai/corso/src/internal/path" "github.com/alcionai/corso/src/pkg/backup/details" ) @@ -14,11 +15,11 @@ type Collection interface { // The channel is closed when there are no more items in the collection or if // an unrecoverable error caused an early termination in the sender. Items() <-chan Stream - // FullPath returns a slice of strings that act as metadata tags for this + // FullPath returns a path struct that acts as a metadata tag for this // DataCollection. Returned items should be ordered from most generic to least // generic. For example, a DataCollection for emails from a specific user - // would be {"", "", "emails"}. - FullPath() []string + // would be {"", "exchange", "", "emails"}. + FullPath() path.Path } // DataStream represents a single item within a DataCollection diff --git a/src/internal/kopia/data_collection.go b/src/internal/kopia/data_collection.go index 7999038bc..6c5b1696c 100644 --- a/src/internal/kopia/data_collection.go +++ b/src/internal/kopia/data_collection.go @@ -2,7 +2,6 @@ package kopia import ( "io" - "strings" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/path" @@ -32,10 +31,8 @@ func (kdc *kopiaDataCollection) Items() <-chan data.Stream { return res } -func (kdc kopiaDataCollection) FullPath() []string { - // TODO(ashmrtn): Update this once data.Collection.FullPath supports - // path.Path. Assumes no adversarial users that use "/" in their folder names. - return strings.Split(kdc.path.String(), "/") +func (kdc kopiaDataCollection) FullPath() path.Path { + return kdc.path } type kopiaDataStream struct { diff --git a/src/internal/kopia/data_collection_test.go b/src/internal/kopia/data_collection_test.go index a45ca6949..97293d2a4 100644 --- a/src/internal/kopia/data_collection_test.go +++ b/src/internal/kopia/data_collection_test.go @@ -27,16 +27,6 @@ func TestKopiaDataCollectionUnitSuite(t *testing.T) { func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() { t := suite.T() - expected := []string{ - "a-tenant", - path.ExchangeService.String(), - "a-user", - path.EmailCategory.String(), - "some", - "path", - "for", - "data", - } b := path.Builder{}.Append("some", "path", "for", "data") pth, err := b.ToDataLayerExchangePathForCategory( @@ -52,8 +42,7 @@ func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() { path: pth, } - // TODO(ashmrtn): Update when data.Collection.FullPath supports path.Path - assert.Equal(t, expected, c.FullPath()) + assert.Equal(t, pth, c.FullPath()) } func (suite *KopiaDataCollectionUnitSuite) TestReturnsStreams() { diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index f8d7974b9..ab205ef53 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -152,19 +152,6 @@ func getStreamItemFunc( return nil } - itemPath, err := path.FromDataLayerPath( - stdpath.Join(streamedEnts.FullPath()...), - false, - ) - if err != nil { - err = errors.Wrap(err, "parsing collection path") - errs = multierror.Append(errs, err) - - logger.Ctx(ctx).Error(err) - - return errs.ErrorOrNil() - } - items := streamedEnts.Items() for { @@ -178,7 +165,7 @@ func getStreamItemFunc( } // For now assuming that item IDs don't need escaping. - itemPath, err := itemPath.Append(e.UUID(), true) + itemPath, err := streamedEnts.FullPath().Append(e.UUID(), true) if err != nil { err = errors.Wrap(err, "getting full item path") errs = multierror.Append(errs, err) @@ -264,7 +251,11 @@ func inflateDirTree( roots := make(map[string]*treeMap) for _, s := range collections { - itemPath := s.FullPath() + if s.FullPath() == nil { + return nil, errors.New("no identifier for collection") + } + + itemPath := s.FullPath().Elements() if len(itemPath) == 0 { return nil, errors.New("no identifier for collection") diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index ee7ab41d9..fb3e87378 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -27,8 +27,8 @@ import ( const ( testTenant = "a-tenant" testUser = "user1" - testInboxDir = "inbox" - testArchiveDir = "archive" + testInboxDir = "Inbox" + testArchiveDir = "Archive" testFileName = "file1" testFileName2 = "file2" testFileName3 = "file3" @@ -75,9 +75,10 @@ func testForFiles( for s := range c.Items() { count++ - fullPath := stdpath.Join(append(c.FullPath(), s.UUID())...) + fullPath, err := c.FullPath().Append(s.UUID(), true) + require.NoError(t, err) - expected, ok := expected[fullPath] + expected, ok := expected[fullPath.String()] require.True(t, ok, "unexpected file with path %q", fullPath) buf, err := ioutil.ReadAll(s.ToReader()) @@ -214,6 +215,23 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { type KopiaUnitSuite struct { suite.Suite + testPath path.Path +} + +func (suite *KopiaUnitSuite) SetupSuite() { + tmp, err := path.FromDataLayerPath( + stdpath.Join( + testTenant, + path.ExchangeService.String(), + testUser, + path.EmailCategory.String(), + testInboxDir, + ), + false, + ) + require.NoError(suite.T(), err) + + suite.testPath = tmp } func TestKopiaUnitSuite(t *testing.T) { @@ -236,6 +254,18 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() { user1 := testUser user2 := "user2" + p2, err := path.FromDataLayerPath( + stdpath.Join( + tenant, + service, + user2, + category, + testInboxDir, + ), + false, + ) + require.NoError(t, err) + expectedFileCount := map[string]int{ user1: 5, user2: 42, @@ -245,11 +275,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() { collections := []data.Collection{ mockconnector.NewMockExchangeCollection( - []string{tenant, service, user1, category, testInboxDir}, + suite.testPath, expectedFileCount[user1], ), mockconnector.NewMockExchangeCollection( - []string{tenant, service, user2, category, testInboxDir}, + p2, expectedFileCount[user2], ), } @@ -301,6 +331,10 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() { func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() { ctx := context.Background() subdir := "subfolder" + + p2, err := suite.testPath.Append(subdir, false) + require.NoError(suite.T(), err) + // Test multiple orders of items because right now order can matter. Both // orders result in a directory structure like: // - a-tenant @@ -319,11 +353,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() { name: "SubdirFirst", layout: []data.Collection{ mockconnector.NewMockExchangeCollection( - []string{testTenant, service, testUser, category, testInboxDir, subdir}, + p2, 5, ), mockconnector.NewMockExchangeCollection( - []string{testTenant, service, testUser, category, testInboxDir}, + suite.testPath, 42, ), }, @@ -332,11 +366,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() { name: "SubdirLast", layout: []data.Collection{ mockconnector.NewMockExchangeCollection( - []string{testTenant, service, testUser, category, testInboxDir}, + suite.testPath, 42, ), mockconnector.NewMockExchangeCollection( - []string{testTenant, service, testUser, category, testInboxDir, subdir}, + p2, 5, ), }, @@ -378,7 +412,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() { } subDirs = append(subDirs, d) - assert.Equal(t, "subfolder", d.Name()) + assert.Equal(t, subdir, d.Name()) } require.Len(t, subDirs, 1) @@ -390,6 +424,14 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() { } func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() { + p2, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory( + "tenant2", + "user2", + path.EmailCategory, + false, + ) + require.NoError(suite.T(), err) + table := []struct { name string layout []data.Collection @@ -397,19 +439,25 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() { { "MultipleRoots", // Directory structure would look like: - // - user1 - // - emails - // - 5 separate files - // - user2 - // - emails - // - 42 separate files + // - tenant1 + // - exchange + // - user1 + // - emails + // - Inbox + // - 5 separate files + // - tenant2 + // - exchange + // - user2 + // - emails + // - Inbox + // - 42 separate files []data.Collection{ mockconnector.NewMockExchangeCollection( - []string{"user1", "emails"}, + suite.testPath, 5, ), mockconnector.NewMockExchangeCollection( - []string{"user2", "emails"}, + p2, 42, ), }, @@ -457,6 +505,9 @@ type KopiaIntegrationSuite struct { suite.Suite w *Wrapper ctx context.Context + + testPath1 path.Path + testPath2 path.Path } func TestKopiaIntegrationSuite(t *testing.T) { @@ -473,6 +524,26 @@ func TestKopiaIntegrationSuite(t *testing.T) { func (suite *KopiaIntegrationSuite) SetupSuite() { _, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...) require.NoError(suite.T(), err) + + tmp, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory( + testTenant, + testUser, + path.EmailCategory, + false, + ) + require.NoError(suite.T(), err) + + suite.testPath1 = tmp + + tmp, err = path.Builder{}.Append(testArchiveDir).ToDataLayerExchangePathForCategory( + testTenant, + testUser, + path.EmailCategory, + false, + ) + require.NoError(suite.T(), err) + + suite.testPath2 = tmp } func (suite *KopiaIntegrationSuite) SetupTest() { @@ -494,11 +565,11 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { collections := []data.Collection{ mockconnector.NewMockExchangeCollection( - []string{"a-tenant", service, "user1", category, testInboxDir}, + suite.testPath1, 5, ), mockconnector.NewMockExchangeCollection( - []string{"a-tenant", service, "user2", category, testInboxDir}, + suite.testPath2, 42, ), } @@ -506,7 +577,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { stats, rp, err := suite.w.BackupCollections(suite.ctx, collections) assert.NoError(t, err) assert.Equal(t, stats.TotalFileCount, 47) - assert.Equal(t, stats.TotalDirectoryCount, 8) + assert.Equal(t, stats.TotalDirectoryCount, 6) assert.Equal(t, stats.IgnoredErrorCount, 0) assert.Equal(t, stats.ErrorCount, 0) assert.False(t, stats.Incomplete) @@ -524,26 +595,14 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { w := &Wrapper{k} - tid := uuid.NewString() - p1 := []string{ - tid, - service, - "uid", - category, - "fid", - } - p2 := []string{ - tid, - service, - "uid2", - category, - "fid", - } - dc1 := mockconnector.NewMockExchangeCollection(p1, 1) - dc2 := mockconnector.NewMockExchangeCollection(p2, 1) + dc1 := mockconnector.NewMockExchangeCollection(suite.testPath1, 1) + dc2 := mockconnector.NewMockExchangeCollection(suite.testPath2, 1) - fp1 := append(p1, dc1.Names[0]) - fp2 := append(p2, dc2.Names[0]) + fp1, err := suite.testPath1.Append(dc1.Names[0], true) + require.NoError(t, err) + + fp2, err := suite.testPath2.Append(dc2.Names[0], true) + require.NoError(t, err) stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}) require.NoError(t, err) @@ -551,14 +610,19 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { require.NoError(t, k.Compression(ctx, "gzip")) expected := map[string][]byte{ - stdpath.Join(fp1...): dc1.Data[0], - stdpath.Join(fp2...): dc2.Data[0], + fp1.String(): dc1.Data[0], + fp2.String(): dc2.Data[0], } result, err := w.RestoreMultipleItems( ctx, string(stats.SnapshotID), - [][]string{fp1, fp2}) + [][]string{ + // TODO(ashmrtn): Remove when selectors passes paths to wrapper for + // restore. + fp1.Elements(), + fp2.Elements(), + }) require.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -568,29 +632,10 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { t := suite.T() - tmpBuilder := path.Builder{}.Append(testInboxDir) - - p1, err := tmpBuilder.ToDataLayerExchangePathForCategory( - testTenant, - testUser, - path.EmailCategory, - false, - ) - require.NoError(t, err) - - tmpBuilder = path.Builder{}.Append(testArchiveDir) - - p2, err := tmpBuilder.ToDataLayerExchangePathForCategory( - testTenant, - testUser, - path.EmailCategory, - false, - ) - require.NoError(t, err) collections := []data.Collection{ &kopiaDataCollection{ - path: p1, + path: suite.testPath1, streams: []data.Stream{ &mockconnector.MockExchangeData{ ID: testFileName, @@ -603,7 +648,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { }, }, &kopiaDataCollection{ - path: p2, + path: suite.testPath2, streams: []data.Stream{ &mockconnector.MockExchangeData{ ID: testFileName3, @@ -644,6 +689,9 @@ type KopiaSimpleRepoIntegrationSuite struct { inboxExpectedFiles map[string][]byte archiveExpectedFiles map[string][]byte allExpectedFiles map[string][]byte + + testPath1 path.Path + testPath2 path.Path } func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) { @@ -660,6 +708,26 @@ func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) { func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { _, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...) require.NoError(suite.T(), err) + + tmp, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory( + testTenant, + testUser, + path.EmailCategory, + false, + ) + require.NoError(suite.T(), err) + + suite.testPath1 = tmp + + tmp, err = path.Builder{}.Append(testArchiveDir).ToDataLayerExchangePathForCategory( + testTenant, + testUser, + path.EmailCategory, + false, + ) + require.NoError(suite.T(), err) + + suite.testPath2 = tmp } func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { @@ -669,29 +737,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { require.NoError(t, err) suite.w = &Wrapper{c} - tmpBuilder := path.Builder{}.Append(testInboxDir) - - p1, err := tmpBuilder.ToDataLayerExchangePathForCategory( - testTenant, - testUser, - path.EmailCategory, - false, - ) - require.NoError(t, err) - - tmpBuilder = path.Builder{}.Append(testArchiveDir) - - p2, err := tmpBuilder.ToDataLayerExchangePathForCategory( - testTenant, - testUser, - path.EmailCategory, - false, - ) - require.NoError(t, err) collections := []data.Collection{ &kopiaDataCollection{ - path: p1, + path: suite.testPath1, streams: []data.Stream{ &mockconnector.MockExchangeData{ ID: testFileName, @@ -704,7 +753,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { }, }, &kopiaDataCollection{ - path: p2, + path: suite.testPath2, streams: []data.Stream{ &mockconnector.MockExchangeData{ ID: testFileName3, @@ -773,7 +822,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupAndRestoreSingleItem() { ) require.NoError(t, err) - assert.Equal(t, c.FullPath(), testPath) + assert.Equal(t, suite.testPath1, c.FullPath()) count := 0 @@ -839,39 +888,31 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() { w := &Wrapper{k} - tid := uuid.NewString() - p1 := []string{ - tid, - service, - "uid", - category, - "fid", - } - p2 := []string{ - tid, - service, - "uid2", - category, - "fid", - } - dc1 := mockconnector.NewMockExchangeCollection(p1, 1) - dc2 := mockconnector.NewMockExchangeCollection(p2, 1) + dc1 := mockconnector.NewMockExchangeCollection(suite.testPath1, 1) + dc2 := mockconnector.NewMockExchangeCollection(suite.testPath2, 1) - fp1 := append(p1, dc1.Names[0]) - fp2 := append(p2, dc2.Names[0]) + fp1, err := suite.testPath1.Append(dc1.Names[0], true) + require.NoError(t, err) + + fp2, err := suite.testPath2.Append(dc2.Names[0], true) + require.NoError(t, err) stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}) require.NoError(t, err) expected := map[string][]byte{ - stdpath.Join(fp1...): dc1.Data[0], - stdpath.Join(fp2...): dc2.Data[0], + fp1.String(): dc1.Data[0], + fp2.String(): dc2.Data[0], } result, err := w.RestoreMultipleItems( ctx, string(stats.SnapshotID), - [][]string{fp1, fp2}) + [][]string{ + // TODO(ashmrtn): Remove when selectors pass path to kopia restore. + fp1.Elements(), + fp2.Elements(), + }) require.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -919,34 +960,18 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors() } } -func (suite *KopiaIntegrationSuite) TestDeleteSnapshot() { +func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() { t := suite.T() - dc1 := mockconnector.NewMockExchangeCollection( - []string{"a-tenant", service, "user1", category, testInboxDir}, - 5, - ) - collections := []data.Collection{ - dc1, - mockconnector.NewMockExchangeCollection( - []string{"a-tenant", service, "user2", category, testInboxDir}, - 42, - ), - } - - bs, _, err := suite.w.BackupCollections(suite.ctx, collections) - require.NoError(t, err) - - snapshotID := bs.SnapshotID - assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, snapshotID)) + assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, string(suite.snapshotID))) // assert the deletion worked - itemPath := []string{"a-tenant", "user1", "emails", dc1.Names[0]} - _, err = suite.w.RestoreSingleItem(suite.ctx, snapshotID, itemPath) + itemPath := append(testPath, testFileName) + _, err := suite.w.RestoreSingleItem(suite.ctx, string(suite.snapshotID), itemPath) assert.Error(t, err, "snapshot should be deleted") } -func (suite *KopiaIntegrationSuite) TestDeleteSnapshot_BadIDs() { +func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot_BadIDs() { table := []struct { name string snapshotID string diff --git a/src/internal/path/path.go b/src/internal/path/path.go index 08cb775d4..246ea80f0 100644 --- a/src/internal/path/path.go +++ b/src/internal/path/path.go @@ -74,6 +74,10 @@ type Path interface { // If removing the right-most element would discard one of the required prefix // elements then an error is returned. Dir() (Path, error) + // Elements returns all the elements in the path. This is a temporary function + // and will likely be updated to handle encoded elements instead of clear-text + // elements in the future. + Elements() []string // Append returns a new Path object with the given element added to the end of // the old Path if possible. If the old Path is an item Path then Append // returns an error.