larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change return value of data.Collection.FullPath to path.Path (#841)

Browse Source 
* Make data.Collection.FullPath return path.Path

* Fixup graph connector code for path struct

* Add Elements call to Path interface

Still should not be used in place of the named functions to get specific
path elements (e.x. ResourceOwner())

* Fixup kopia wrapper path handling

Mostly removing shim code and minor fixup for building the directory
tree.

* All the test fixes
This commit is contained in:
ashmrtn 2022-09-14 10:01:11 -07:00 committed by GitHub
parent 59a6bc672a
commit 5ae8259fd5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 198 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -8,7 +8,6 @@ import (
"context"
"fmt"
"io"
"strings"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
kw "github.com/microsoft/kiota-serialization-json-go"
@ -108,12 +107,8 @@ func GetQueryAndSerializeFunc(optID optionIdentifier) (GraphRetrievalFunc, Graph
}
// FullPath returns the Collection's fullPath []string
func (col *Collection) FullPath() []string {
// TODO(ashmrtn): Remove this when data.Collection.FullPath returns a
// path.Path. This assumes we don't have adversarial users that use '/' in
// their folder names.
r := col.fullPath.String()
return strings.Split(r, "/")
func (col *Collection) FullPath() path.Path {
return col.fullPath
}
// populateByOptionIdentifier is a utility function that uses col.collectionType to be able to serialize

16
src/internal/connector/exchange/exchange_data_collection_test.go
View File

@ -51,13 +51,6 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() {
tenant := "a-tenant"
user := "a-user"
folder := "a-folder"
expected := []string{
tenant,
path.ExchangeService.String(),
user,
path.EmailCategory.String(),
folder,
}
fullPath, err := path.Builder{}.Append(folder).ToDataLayerExchangePathForCategory(
tenant,
@ -72,8 +65,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() {
fullPath: fullPath,
}
// TODO(ashmrtn): Update when data.Collection.FullPath returns path.Path.
assert.Equal(t, expected, edc.FullPath())
assert.Equal(t, fullPath, edc.FullPath())
}
func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchangeDataCollection() {
@ -95,11 +87,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchange
fullPath: fullPath,
}
suite.Equal(name, edc.user)
suite.Contains(edc.FullPath(), fullPath.Tenant())
suite.Contains(edc.FullPath(), fullPath.Service().String())
suite.Contains(edc.FullPath(), fullPath.Category().String())
suite.Contains(edc.FullPath(), fullPath.ResourceOwner())
suite.Contains(edc.FullPath(), fullPath.Folder())
suite.Equal(fullPath, edc.FullPath())
}
func (suite *ExchangeDataCollectionSuite) TestExchangeCollection_AddJob() {

5
src/internal/connector/exchange/iterators_test.go
View File

@ -190,10 +190,9 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() {
}
for _, c := range collections {
// TODO(ashmrtn): Update these checks when collections support path.Path.
require.Greater(t, len(c.FullPath()), 4)
require.NotEmpty(t, c.FullPath().Folder())
folder := c.FullPath()[4]
folder := c.FullPath().Folder()
if _, ok := test.folderNames[folder]; ok {
delete(test.folderNames, folder)
}

23
src/internal/connector/graph_connector.go
View File

@ -6,7 +6,6 @@ import (
"bytes"
"context"
"fmt"
stdpath "path"
"sync"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
@ -259,24 +258,14 @@ func (gc *GraphConnector) RestoreDataCollections(
for _, dc := range dcs {
var (
items = dc.Items()
exit bool
items = dc.Items()
directory = dc.FullPath()
service = directory.Service()
category = directory.Category()
user = directory.ResourceOwner()
exit bool
)
// TODO(ashmrtn): Remove this when data.Collection.FullPath supports path.Path
directory, err := path.FromDataLayerPath(
stdpath.Join(dc.FullPath()...),
false,
)
if err != nil {
errs = support.WrapAndAppend("parsing Collection path", err, errs)
continue
}
category := directory.Category()
user := directory.ResourceOwner()
service := directory.Service()
// Check whether restoring data into the specified service is supported
switch service {
case path.ExchangeService:

10
src/internal/connector/graph_connector_disconnected_test.go
View File

@ -10,9 +10,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
@ -86,14 +84,6 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() {
suite.Contains(last, "Foley")
}
func (suite *DisconnectedGraphConnectorSuite) TestInterfaceAlignment() {
var dc data.Collection
concrete := mockconnector.NewMockExchangeCollection([]string{"a", "path"}, 1)
dc = concrete
assert.NotNil(suite.T(), dc)
}
func statusTestTask(gc *GraphConnector, objects, success, folder int) {
status := support.CreateStatus(
context.Background(),

13
src/internal/connector/graph_connector_test.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"context"
"fmt"
"strings"
"testing"
"time"
@ -99,7 +98,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
// Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
for _, collection := range collectionList {
temp := collection.Items()
testName := collection.FullPath()[2]
testName := collection.FullPath().ResourceOwner()
streams[testName] = temp
}
@ -116,9 +115,6 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
}
})
}
exchangeData := collectionList[0]
suite.Greater(len(exchangeData.FullPath()), 2)
}
// TestMailSerializationRegression verifies that all mail data stored in the
@ -139,8 +135,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
require.NoError(t, err)
for _, edc := range collection {
testName := strings.Join(edc.FullPath(), " ")
suite.T().Run(testName, func(t *testing.T) {
suite.T().Run(edc.FullPath().String(), func(t *testing.T) {
streamChannel := edc.Items()
// Verify that each message can be restored
for stream := range streamChannel {
@ -181,7 +176,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
number := 0
for _, edc := range collections {
testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath()[1], number)
testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath().ResourceOwner(), number)
suite.T().Run(testName, func(t *testing.T) {
streamChannel := edc.Items()
for stream := range streamChannel {
@ -220,7 +215,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
number := 0
for stream := range streamChannel {
testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[2], number)
testName := fmt.Sprintf("%s_Event_%d", edc.FullPath().ResourceOwner(), number)
suite.T().Run(testName, func(t *testing.T) {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())

9
src/internal/connector/mockconnector/mock_data_collection.go
View File

@ -10,12 +10,13 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/path"
"github.com/alcionai/corso/src/pkg/backup/details"
)
// MockExchangeDataCollection represents a mock exchange mailbox
type MockExchangeDataCollection struct {
fullPath []string
fullPath path.Path
messageCount int
Data [][]byte
Names []string
@ -29,7 +30,7 @@ var (
// NewMockExchangeDataCollection creates an data collection that will return the specified number of
// mock messages when iterated. Exchange type mail
func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn int) *MockExchangeDataCollection {
func NewMockExchangeCollection(pathRepresentation path.Path, numMessagesToReturn int) *MockExchangeDataCollection {
c := &MockExchangeDataCollection{
fullPath: pathRepresentation,
messageCount: numMessagesToReturn,
@ -46,8 +47,8 @@ func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn
return c
}
func (medc *MockExchangeDataCollection) FullPath() []string {
return append([]string{}, medc.fullPath...)
func (medc *MockExchangeDataCollection) FullPath() path.Path {
return medc.fullPath
}
// Items returns a channel that has the next items in the collection. The

4
src/internal/connector/mockconnector/mock_data_collection_test.go
View File

@ -24,7 +24,7 @@ func TestMockExchangeCollectionSuite(t *testing.T) {
}
func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() {
mdc := mockconnector.NewMockExchangeCollection([]string{"foo", "bar"}, 2)
mdc := mockconnector.NewMockExchangeCollection(nil, 2)
messagesRead := 0
@ -41,7 +41,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() {
// functions by verifying no failures on (de)serializing steps using kiota serialization library
func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection_NewExchangeCollectionMail_Hydration() {
t := suite.T()
mdc := mockconnector.NewMockExchangeCollection([]string{"foo", "bar"}, 3)
mdc := mockconnector.NewMockExchangeCollection(nil, 3)
buf := &bytes.Buffer{}
for stream := range mdc.Items() {

7
src/internal/connector/onedrive/collection.go
View File

@ -4,7 +4,6 @@ package onedrive
import (
"context"
"io"
"strings"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
@ -80,10 +79,8 @@ func (oc *Collection) Items() <-chan data.Stream {
return oc.data
}
func (oc *Collection) FullPath() []string {
// TODO(ashmrtn): Update this when data.Collection.FullPath has support for
// path.Path.
return strings.Split(oc.folderPath.String(), "/")
func (oc *Collection) FullPath() path.Path {
return oc.folderPath
}
// Item represents a single item retrieved from OneDrive

7
src/internal/connector/onedrive/collection_test.go
View File

@ -5,7 +5,6 @@ import (
"context"
"errors"
"io"
"strings"
"sync"
"testing"
@ -64,11 +63,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
require.NotNil(t, coll)
assert.Equal(
t,
strings.Split(folderPath.String(), "/"),
coll.FullPath(),
)
assert.Equal(t, folderPath, coll.FullPath())
testItemID := "fakeItemID"
testItemName := "itemName"

7
src/internal/data/data_collection.go
View File

@ -3,6 +3,7 @@ package data
import (
"io"
"github.com/alcionai/corso/src/internal/path"
"github.com/alcionai/corso/src/pkg/backup/details"
)
@ -14,11 +15,11 @@ type Collection interface {
// The channel is closed when there are no more items in the collection or if
// an unrecoverable error caused an early termination in the sender.
Items() <-chan Stream
// FullPath returns a slice of strings that act as metadata tags for this
// FullPath returns a path struct that acts as a metadata tag for this
// DataCollection. Returned items should be ordered from most generic to least
// generic. For example, a DataCollection for emails from a specific user
// would be {"<tenant id>", "<user ID>", "emails"}.
FullPath() []string
// would be {"<tenant id>", "exchange", "<user ID>", "emails"}.
FullPath() path.Path
}
// DataStream represents a single item within a DataCollection

7
src/internal/kopia/data_collection.go
View File

@ -2,7 +2,6 @@ package kopia
import (
"io"
"strings"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/path"
@ -32,10 +31,8 @@ func (kdc *kopiaDataCollection) Items() <-chan data.Stream {
return res
}
func (kdc kopiaDataCollection) FullPath() []string {
// TODO(ashmrtn): Update this once data.Collection.FullPath supports
// path.Path. Assumes no adversarial users that use "/" in their folder names.
return strings.Split(kdc.path.String(), "/")
func (kdc kopiaDataCollection) FullPath() path.Path {
return kdc.path
}
type kopiaDataStream struct {

13
src/internal/kopia/data_collection_test.go
View File

@ -27,16 +27,6 @@ func TestKopiaDataCollectionUnitSuite(t *testing.T) {
func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() {
t := suite.T()
expected := []string{
"a-tenant",
path.ExchangeService.String(),
"a-user",
path.EmailCategory.String(),
"some",
"path",
"for",
"data",
}
b := path.Builder{}.Append("some", "path", "for", "data")
pth, err := b.ToDataLayerExchangePathForCategory(
@ -52,8 +42,7 @@ func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() {
path: pth,
}
// TODO(ashmrtn): Update when data.Collection.FullPath supports path.Path
assert.Equal(t, expected, c.FullPath())
assert.Equal(t, pth, c.FullPath())
}
func (suite *KopiaDataCollectionUnitSuite) TestReturnsStreams() {

21
src/internal/kopia/wrapper.go
View File

@ -152,19 +152,6 @@ func getStreamItemFunc(
return nil
}
itemPath, err := path.FromDataLayerPath(
stdpath.Join(streamedEnts.FullPath()...),
false,
)
if err != nil {
err = errors.Wrap(err, "parsing collection path")
errs = multierror.Append(errs, err)
logger.Ctx(ctx).Error(err)
return errs.ErrorOrNil()
}
items := streamedEnts.Items()
for {
@ -178,7 +165,7 @@ func getStreamItemFunc(
}
// For now assuming that item IDs don't need escaping.
itemPath, err := itemPath.Append(e.UUID(), true)
itemPath, err := streamedEnts.FullPath().Append(e.UUID(), true)
if err != nil {
err = errors.Wrap(err, "getting full item path")
errs = multierror.Append(errs, err)
@ -264,7 +251,11 @@ func inflateDirTree(
roots := make(map[string]*treeMap)
for _, s := range collections {
itemPath := s.FullPath()
if s.FullPath() == nil {
return nil, errors.New("no identifier for collection")
}
itemPath := s.FullPath().Elements()
if len(itemPath) == 0 {
return nil, errors.New("no identifier for collection")

285
src/internal/kopia/wrapper_test.go
View File

@ -27,8 +27,8 @@ import (
const (
testTenant = "a-tenant"
testUser = "user1"
testInboxDir = "inbox"
testArchiveDir = "archive"
testInboxDir = "Inbox"
testArchiveDir = "Archive"
testFileName = "file1"
testFileName2 = "file2"
testFileName3 = "file3"
@ -75,9 +75,10 @@ func testForFiles(
for s := range c.Items() {
count++
fullPath := stdpath.Join(append(c.FullPath(), s.UUID())...)
fullPath, err := c.FullPath().Append(s.UUID(), true)
require.NoError(t, err)
expected, ok := expected[fullPath]
expected, ok := expected[fullPath.String()]
require.True(t, ok, "unexpected file with path %q", fullPath)
buf, err := ioutil.ReadAll(s.ToReader())
@ -214,6 +215,23 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
type KopiaUnitSuite struct {
suite.Suite
testPath path.Path
}
func (suite *KopiaUnitSuite) SetupSuite() {
tmp, err := path.FromDataLayerPath(
stdpath.Join(
testTenant,
path.ExchangeService.String(),
testUser,
path.EmailCategory.String(),
testInboxDir,
),
false,
)
require.NoError(suite.T(), err)
suite.testPath = tmp
}
func TestKopiaUnitSuite(t *testing.T) {
@ -236,6 +254,18 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
user1 := testUser
user2 := "user2"
p2, err := path.FromDataLayerPath(
stdpath.Join(
tenant,
service,
user2,
category,
testInboxDir,
),
false,
)
require.NoError(t, err)
expectedFileCount := map[string]int{
user1: 5,
user2: 42,
@ -245,11 +275,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
collections := []data.Collection{
mockconnector.NewMockExchangeCollection(
[]string{tenant, service, user1, category, testInboxDir},
suite.testPath,
expectedFileCount[user1],
),
mockconnector.NewMockExchangeCollection(
[]string{tenant, service, user2, category, testInboxDir},
p2,
expectedFileCount[user2],
),
}
@ -301,6 +331,10 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
ctx := context.Background()
subdir := "subfolder"
p2, err := suite.testPath.Append(subdir, false)
require.NoError(suite.T(), err)
// Test multiple orders of items because right now order can matter. Both
// orders result in a directory structure like:
// - a-tenant
@ -319,11 +353,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
name: "SubdirFirst",
layout: []data.Collection{
mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir, subdir},
p2,
5,
),
mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir},
suite.testPath,
42,
),
},
@ -332,11 +366,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
name: "SubdirLast",
layout: []data.Collection{
mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir},
suite.testPath,
42,
),
mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir, subdir},
p2,
5,
),
},
@ -378,7 +412,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
}
subDirs = append(subDirs, d)
assert.Equal(t, "subfolder", d.Name())
assert.Equal(t, subdir, d.Name())
}
require.Len(t, subDirs, 1)
@ -390,6 +424,14 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
}
func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() {
p2, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory(
"tenant2",
"user2",
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
table := []struct {
name string
layout []data.Collection
@ -397,19 +439,25 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() {
{
"MultipleRoots",
// Directory structure would look like:
// - user1
// - emails
// - 5 separate files
// - user2
// - emails
// - 42 separate files
// - tenant1
// - exchange
// - user1
// - emails
// - Inbox
// - 5 separate files
// - tenant2
// - exchange
// - user2
// - emails
// - Inbox
// - 42 separate files
[]data.Collection{
mockconnector.NewMockExchangeCollection(
[]string{"user1", "emails"},
suite.testPath,
5,
),
mockconnector.NewMockExchangeCollection(
[]string{"user2", "emails"},
p2,
42,
),
},
@ -457,6 +505,9 @@ type KopiaIntegrationSuite struct {
suite.Suite
w *Wrapper
ctx context.Context
testPath1 path.Path
testPath2 path.Path
}
func TestKopiaIntegrationSuite(t *testing.T) {
@ -473,6 +524,26 @@ func TestKopiaIntegrationSuite(t *testing.T) {
func (suite *KopiaIntegrationSuite) SetupSuite() {
_, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...)
require.NoError(suite.T(), err)
tmp, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath1 = tmp
tmp, err = path.Builder{}.Append(testArchiveDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath2 = tmp
}
func (suite *KopiaIntegrationSuite) SetupTest() {
@ -494,11 +565,11 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
collections := []data.Collection{
mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user1", category, testInboxDir},
suite.testPath1,
5,
),
mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user2", category, testInboxDir},
suite.testPath2,
42,
),
}
@ -506,7 +577,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
stats, rp, err := suite.w.BackupCollections(suite.ctx, collections)
assert.NoError(t, err)
assert.Equal(t, stats.TotalFileCount, 47)
assert.Equal(t, stats.TotalDirectoryCount, 8)
assert.Equal(t, stats.TotalDirectoryCount, 6)
assert.Equal(t, stats.IgnoredErrorCount, 0)
assert.Equal(t, stats.ErrorCount, 0)
assert.False(t, stats.Incomplete)
@ -524,26 +595,14 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
w := &Wrapper{k}
tid := uuid.NewString()
p1 := []string{
tid,
service,
"uid",
category,
"fid",
}
p2 := []string{
tid,
service,
"uid2",
category,
"fid",
}
dc1 := mockconnector.NewMockExchangeCollection(p1, 1)
dc2 := mockconnector.NewMockExchangeCollection(p2, 1)
dc1 := mockconnector.NewMockExchangeCollection(suite.testPath1, 1)
dc2 := mockconnector.NewMockExchangeCollection(suite.testPath2, 1)
fp1 := append(p1, dc1.Names[0])
fp2 := append(p2, dc2.Names[0])
fp1, err := suite.testPath1.Append(dc1.Names[0], true)
require.NoError(t, err)
fp2, err := suite.testPath2.Append(dc2.Names[0], true)
require.NoError(t, err)
stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2})
require.NoError(t, err)
@ -551,14 +610,19 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
require.NoError(t, k.Compression(ctx, "gzip"))
expected := map[string][]byte{
stdpath.Join(fp1...): dc1.Data[0],
stdpath.Join(fp2...): dc2.Data[0],
fp1.String(): dc1.Data[0],
fp2.String(): dc2.Data[0],
}
result, err := w.RestoreMultipleItems(
ctx,
string(stats.SnapshotID),
[][]string{fp1, fp2})
[][]string{
// TODO(ashmrtn): Remove when selectors passes paths to wrapper for
// restore.
fp1.Elements(),
fp2.Elements(),
})
require.NoError(t, err)
assert.Equal(t, 2, len(result))
@ -568,29 +632,10 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
t := suite.T()
tmpBuilder := path.Builder{}.Append(testInboxDir)
p1, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
tmpBuilder = path.Builder{}.Append(testArchiveDir)
p2, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
collections := []data.Collection{
&kopiaDataCollection{
path: p1,
path: suite.testPath1,
streams: []data.Stream{
&mockconnector.MockExchangeData{
ID: testFileName,
@ -603,7 +648,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
},
},
&kopiaDataCollection{
path: p2,
path: suite.testPath2,
streams: []data.Stream{
&mockconnector.MockExchangeData{
ID: testFileName3,
@ -644,6 +689,9 @@ type KopiaSimpleRepoIntegrationSuite struct {
inboxExpectedFiles map[string][]byte
archiveExpectedFiles map[string][]byte
allExpectedFiles map[string][]byte
testPath1 path.Path
testPath2 path.Path
}
func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) {
@ -660,6 +708,26 @@ func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) {
func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() {
_, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...)
require.NoError(suite.T(), err)
tmp, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath1 = tmp
tmp, err = path.Builder{}.Append(testArchiveDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath2 = tmp
}
func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
@ -669,29 +737,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
require.NoError(t, err)
suite.w = &Wrapper{c}
tmpBuilder := path.Builder{}.Append(testInboxDir)
p1, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
tmpBuilder = path.Builder{}.Append(testArchiveDir)
p2, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
collections := []data.Collection{
&kopiaDataCollection{
path: p1,
path: suite.testPath1,
streams: []data.Stream{
&mockconnector.MockExchangeData{
ID: testFileName,
@ -704,7 +753,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
},
},
&kopiaDataCollection{
path: p2,
path: suite.testPath2,
streams: []data.Stream{
&mockconnector.MockExchangeData{
ID: testFileName3,
@ -773,7 +822,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupAndRestoreSingleItem() {
)
require.NoError(t, err)
assert.Equal(t, c.FullPath(), testPath)
assert.Equal(t, suite.testPath1, c.FullPath())
count := 0
@ -839,39 +888,31 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() {
w := &Wrapper{k}
tid := uuid.NewString()
p1 := []string{
tid,
service,
"uid",
category,
"fid",
}
p2 := []string{
tid,
service,
"uid2",
category,
"fid",
}
dc1 := mockconnector.NewMockExchangeCollection(p1, 1)
dc2 := mockconnector.NewMockExchangeCollection(p2, 1)
dc1 := mockconnector.NewMockExchangeCollection(suite.testPath1, 1)
dc2 := mockconnector.NewMockExchangeCollection(suite.testPath2, 1)
fp1 := append(p1, dc1.Names[0])
fp2 := append(p2, dc2.Names[0])
fp1, err := suite.testPath1.Append(dc1.Names[0], true)
require.NoError(t, err)
fp2, err := suite.testPath2.Append(dc2.Names[0], true)
require.NoError(t, err)
stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2})
require.NoError(t, err)
expected := map[string][]byte{
stdpath.Join(fp1...): dc1.Data[0],
stdpath.Join(fp2...): dc2.Data[0],
fp1.String(): dc1.Data[0],
fp2.String(): dc2.Data[0],
}
result, err := w.RestoreMultipleItems(
ctx,
string(stats.SnapshotID),
[][]string{fp1, fp2})
[][]string{
// TODO(ashmrtn): Remove when selectors pass path to kopia restore.
fp1.Elements(),
fp2.Elements(),
})
require.NoError(t, err)
assert.Equal(t, 2, len(result))
@ -919,34 +960,18 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors()
}
}
func (suite *KopiaIntegrationSuite) TestDeleteSnapshot() {
func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() {
t := suite.T()
dc1 := mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user1", category, testInboxDir},
5,
)
collections := []data.Collection{
dc1,
mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user2", category, testInboxDir},
42,
),
}
bs, _, err := suite.w.BackupCollections(suite.ctx, collections)
require.NoError(t, err)
snapshotID := bs.SnapshotID
assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, snapshotID))
assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, string(suite.snapshotID)))
// assert the deletion worked
itemPath := []string{"a-tenant", "user1", "emails", dc1.Names[0]}
_, err = suite.w.RestoreSingleItem(suite.ctx, snapshotID, itemPath)
itemPath := append(testPath, testFileName)
_, err := suite.w.RestoreSingleItem(suite.ctx, string(suite.snapshotID), itemPath)
assert.Error(t, err, "snapshot should be deleted")
}
func (suite *KopiaIntegrationSuite) TestDeleteSnapshot_BadIDs() {
func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot_BadIDs() {
table := []struct {
name string
snapshotID string

4
src/internal/path/path.go
View File

@ -74,6 +74,10 @@ type Path interface {
// If removing the right-most element would discard one of the required prefix
// elements then an error is returned.
Dir() (Path, error)
// Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text
// elements in the future.
Elements() []string
// Append returns a new Path object with the given element added to the end of
// the old Path if possible. If the old Path is an item Path then Append
// returns an error.