Change return value of data.Collection.FullPath to path.Path (#841)

* Make data.Collection.FullPath return path.Path

* Fixup graph connector code for path struct

* Add Elements call to Path interface

Still should not be used in place of the named functions to get specific
path elements (e.x. ResourceOwner())

* Fixup kopia wrapper path handling

Mostly removing shim code and minor fixup for building the directory
tree.

* All the test fixes
This commit is contained in:
ashmrtn 2022-09-14 10:01:11 -07:00 committed by GitHub
parent 59a6bc672a
commit 5ae8259fd5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 198 additions and 242 deletions

View File

@ -8,7 +8,6 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"strings"
absser "github.com/microsoft/kiota-abstractions-go/serialization" absser "github.com/microsoft/kiota-abstractions-go/serialization"
kw "github.com/microsoft/kiota-serialization-json-go" kw "github.com/microsoft/kiota-serialization-json-go"
@ -108,12 +107,8 @@ func GetQueryAndSerializeFunc(optID optionIdentifier) (GraphRetrievalFunc, Graph
} }
// FullPath returns the Collection's fullPath []string // FullPath returns the Collection's fullPath []string
func (col *Collection) FullPath() []string { func (col *Collection) FullPath() path.Path {
// TODO(ashmrtn): Remove this when data.Collection.FullPath returns a return col.fullPath
// path.Path. This assumes we don't have adversarial users that use '/' in
// their folder names.
r := col.fullPath.String()
return strings.Split(r, "/")
} }
// populateByOptionIdentifier is a utility function that uses col.collectionType to be able to serialize // populateByOptionIdentifier is a utility function that uses col.collectionType to be able to serialize

View File

@ -51,13 +51,6 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() {
tenant := "a-tenant" tenant := "a-tenant"
user := "a-user" user := "a-user"
folder := "a-folder" folder := "a-folder"
expected := []string{
tenant,
path.ExchangeService.String(),
user,
path.EmailCategory.String(),
folder,
}
fullPath, err := path.Builder{}.Append(folder).ToDataLayerExchangePathForCategory( fullPath, err := path.Builder{}.Append(folder).ToDataLayerExchangePathForCategory(
tenant, tenant,
@ -72,8 +65,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() {
fullPath: fullPath, fullPath: fullPath,
} }
// TODO(ashmrtn): Update when data.Collection.FullPath returns path.Path. assert.Equal(t, fullPath, edc.FullPath())
assert.Equal(t, expected, edc.FullPath())
} }
func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchangeDataCollection() { func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchangeDataCollection() {
@ -95,11 +87,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchange
fullPath: fullPath, fullPath: fullPath,
} }
suite.Equal(name, edc.user) suite.Equal(name, edc.user)
suite.Contains(edc.FullPath(), fullPath.Tenant()) suite.Equal(fullPath, edc.FullPath())
suite.Contains(edc.FullPath(), fullPath.Service().String())
suite.Contains(edc.FullPath(), fullPath.Category().String())
suite.Contains(edc.FullPath(), fullPath.ResourceOwner())
suite.Contains(edc.FullPath(), fullPath.Folder())
} }
func (suite *ExchangeDataCollectionSuite) TestExchangeCollection_AddJob() { func (suite *ExchangeDataCollectionSuite) TestExchangeCollection_AddJob() {

View File

@ -190,10 +190,9 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() {
} }
for _, c := range collections { for _, c := range collections {
// TODO(ashmrtn): Update these checks when collections support path.Path. require.NotEmpty(t, c.FullPath().Folder())
require.Greater(t, len(c.FullPath()), 4)
folder := c.FullPath()[4] folder := c.FullPath().Folder()
if _, ok := test.folderNames[folder]; ok { if _, ok := test.folderNames[folder]; ok {
delete(test.folderNames, folder) delete(test.folderNames, folder)
} }

View File

@ -6,7 +6,6 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
stdpath "path"
"sync" "sync"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
@ -259,24 +258,14 @@ func (gc *GraphConnector) RestoreDataCollections(
for _, dc := range dcs { for _, dc := range dcs {
var ( var (
items = dc.Items() items = dc.Items()
exit bool directory = dc.FullPath()
service = directory.Service()
category = directory.Category()
user = directory.ResourceOwner()
exit bool
) )
// TODO(ashmrtn): Remove this when data.Collection.FullPath supports path.Path
directory, err := path.FromDataLayerPath(
stdpath.Join(dc.FullPath()...),
false,
)
if err != nil {
errs = support.WrapAndAppend("parsing Collection path", err, errs)
continue
}
category := directory.Category()
user := directory.ResourceOwner()
service := directory.Service()
// Check whether restoring data into the specified service is supported // Check whether restoring data into the specified service is supported
switch service { switch service {
case path.ExchangeService: case path.ExchangeService:

View File

@ -10,9 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
@ -86,14 +84,6 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() {
suite.Contains(last, "Foley") suite.Contains(last, "Foley")
} }
func (suite *DisconnectedGraphConnectorSuite) TestInterfaceAlignment() {
var dc data.Collection
concrete := mockconnector.NewMockExchangeCollection([]string{"a", "path"}, 1)
dc = concrete
assert.NotNil(suite.T(), dc)
}
func statusTestTask(gc *GraphConnector, objects, success, folder int) { func statusTestTask(gc *GraphConnector, objects, success, folder int) {
status := support.CreateStatus( status := support.CreateStatus(
context.Background(), context.Background(),

View File

@ -4,7 +4,6 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"strings"
"testing" "testing"
"time" "time"
@ -99,7 +98,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
// Verify Items() call returns an iterable channel(e.g. a channel that has been closed) // Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
for _, collection := range collectionList { for _, collection := range collectionList {
temp := collection.Items() temp := collection.Items()
testName := collection.FullPath()[2] testName := collection.FullPath().ResourceOwner()
streams[testName] = temp streams[testName] = temp
} }
@ -116,9 +115,6 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
} }
}) })
} }
exchangeData := collectionList[0]
suite.Greater(len(exchangeData.FullPath()), 2)
} }
// TestMailSerializationRegression verifies that all mail data stored in the // TestMailSerializationRegression verifies that all mail data stored in the
@ -139,8 +135,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
require.NoError(t, err) require.NoError(t, err)
for _, edc := range collection { for _, edc := range collection {
testName := strings.Join(edc.FullPath(), " ") suite.T().Run(edc.FullPath().String(), func(t *testing.T) {
suite.T().Run(testName, func(t *testing.T) {
streamChannel := edc.Items() streamChannel := edc.Items()
// Verify that each message can be restored // Verify that each message can be restored
for stream := range streamChannel { for stream := range streamChannel {
@ -181,7 +176,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
number := 0 number := 0
for _, edc := range collections { for _, edc := range collections {
testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath()[1], number) testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath().ResourceOwner(), number)
suite.T().Run(testName, func(t *testing.T) { suite.T().Run(testName, func(t *testing.T) {
streamChannel := edc.Items() streamChannel := edc.Items()
for stream := range streamChannel { for stream := range streamChannel {
@ -220,7 +215,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
number := 0 number := 0
for stream := range streamChannel { for stream := range streamChannel {
testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[2], number) testName := fmt.Sprintf("%s_Event_%d", edc.FullPath().ResourceOwner(), number)
suite.T().Run(testName, func(t *testing.T) { suite.T().Run(testName, func(t *testing.T) {
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader()) read, err := buf.ReadFrom(stream.ToReader())

View File

@ -10,12 +10,13 @@ import (
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/path"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
) )
// MockExchangeDataCollection represents a mock exchange mailbox // MockExchangeDataCollection represents a mock exchange mailbox
type MockExchangeDataCollection struct { type MockExchangeDataCollection struct {
fullPath []string fullPath path.Path
messageCount int messageCount int
Data [][]byte Data [][]byte
Names []string Names []string
@ -29,7 +30,7 @@ var (
// NewMockExchangeDataCollection creates an data collection that will return the specified number of // NewMockExchangeDataCollection creates an data collection that will return the specified number of
// mock messages when iterated. Exchange type mail // mock messages when iterated. Exchange type mail
func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn int) *MockExchangeDataCollection { func NewMockExchangeCollection(pathRepresentation path.Path, numMessagesToReturn int) *MockExchangeDataCollection {
c := &MockExchangeDataCollection{ c := &MockExchangeDataCollection{
fullPath: pathRepresentation, fullPath: pathRepresentation,
messageCount: numMessagesToReturn, messageCount: numMessagesToReturn,
@ -46,8 +47,8 @@ func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn
return c return c
} }
func (medc *MockExchangeDataCollection) FullPath() []string { func (medc *MockExchangeDataCollection) FullPath() path.Path {
return append([]string{}, medc.fullPath...) return medc.fullPath
} }
// Items returns a channel that has the next items in the collection. The // Items returns a channel that has the next items in the collection. The

View File

@ -24,7 +24,7 @@ func TestMockExchangeCollectionSuite(t *testing.T) {
} }
func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() { func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() {
mdc := mockconnector.NewMockExchangeCollection([]string{"foo", "bar"}, 2) mdc := mockconnector.NewMockExchangeCollection(nil, 2)
messagesRead := 0 messagesRead := 0
@ -41,7 +41,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() {
// functions by verifying no failures on (de)serializing steps using kiota serialization library // functions by verifying no failures on (de)serializing steps using kiota serialization library
func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection_NewExchangeCollectionMail_Hydration() { func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection_NewExchangeCollectionMail_Hydration() {
t := suite.T() t := suite.T()
mdc := mockconnector.NewMockExchangeCollection([]string{"foo", "bar"}, 3) mdc := mockconnector.NewMockExchangeCollection(nil, 3)
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
for stream := range mdc.Items() { for stream := range mdc.Items() {

View File

@ -4,7 +4,6 @@ package onedrive
import ( import (
"context" "context"
"io" "io"
"strings"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
@ -80,10 +79,8 @@ func (oc *Collection) Items() <-chan data.Stream {
return oc.data return oc.data
} }
func (oc *Collection) FullPath() []string { func (oc *Collection) FullPath() path.Path {
// TODO(ashmrtn): Update this when data.Collection.FullPath has support for return oc.folderPath
// path.Path.
return strings.Split(oc.folderPath.String(), "/")
} }
// Item represents a single item retrieved from OneDrive // Item represents a single item retrieved from OneDrive

View File

@ -5,7 +5,6 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"strings"
"sync" "sync"
"testing" "testing"
@ -64,11 +63,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus)) coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
require.NotNil(t, coll) require.NotNil(t, coll)
assert.Equal( assert.Equal(t, folderPath, coll.FullPath())
t,
strings.Split(folderPath.String(), "/"),
coll.FullPath(),
)
testItemID := "fakeItemID" testItemID := "fakeItemID"
testItemName := "itemName" testItemName := "itemName"

View File

@ -3,6 +3,7 @@ package data
import ( import (
"io" "io"
"github.com/alcionai/corso/src/internal/path"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
) )
@ -14,11 +15,11 @@ type Collection interface {
// The channel is closed when there are no more items in the collection or if // The channel is closed when there are no more items in the collection or if
// an unrecoverable error caused an early termination in the sender. // an unrecoverable error caused an early termination in the sender.
Items() <-chan Stream Items() <-chan Stream
// FullPath returns a slice of strings that act as metadata tags for this // FullPath returns a path struct that acts as a metadata tag for this
// DataCollection. Returned items should be ordered from most generic to least // DataCollection. Returned items should be ordered from most generic to least
// generic. For example, a DataCollection for emails from a specific user // generic. For example, a DataCollection for emails from a specific user
// would be {"<tenant id>", "<user ID>", "emails"}. // would be {"<tenant id>", "exchange", "<user ID>", "emails"}.
FullPath() []string FullPath() path.Path
} }
// DataStream represents a single item within a DataCollection // DataStream represents a single item within a DataCollection

View File

@ -2,7 +2,6 @@ package kopia
import ( import (
"io" "io"
"strings"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/path" "github.com/alcionai/corso/src/internal/path"
@ -32,10 +31,8 @@ func (kdc *kopiaDataCollection) Items() <-chan data.Stream {
return res return res
} }
func (kdc kopiaDataCollection) FullPath() []string { func (kdc kopiaDataCollection) FullPath() path.Path {
// TODO(ashmrtn): Update this once data.Collection.FullPath supports return kdc.path
// path.Path. Assumes no adversarial users that use "/" in their folder names.
return strings.Split(kdc.path.String(), "/")
} }
type kopiaDataStream struct { type kopiaDataStream struct {

View File

@ -27,16 +27,6 @@ func TestKopiaDataCollectionUnitSuite(t *testing.T) {
func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() { func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() {
t := suite.T() t := suite.T()
expected := []string{
"a-tenant",
path.ExchangeService.String(),
"a-user",
path.EmailCategory.String(),
"some",
"path",
"for",
"data",
}
b := path.Builder{}.Append("some", "path", "for", "data") b := path.Builder{}.Append("some", "path", "for", "data")
pth, err := b.ToDataLayerExchangePathForCategory( pth, err := b.ToDataLayerExchangePathForCategory(
@ -52,8 +42,7 @@ func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() {
path: pth, path: pth,
} }
// TODO(ashmrtn): Update when data.Collection.FullPath supports path.Path assert.Equal(t, pth, c.FullPath())
assert.Equal(t, expected, c.FullPath())
} }
func (suite *KopiaDataCollectionUnitSuite) TestReturnsStreams() { func (suite *KopiaDataCollectionUnitSuite) TestReturnsStreams() {

View File

@ -152,19 +152,6 @@ func getStreamItemFunc(
return nil return nil
} }
itemPath, err := path.FromDataLayerPath(
stdpath.Join(streamedEnts.FullPath()...),
false,
)
if err != nil {
err = errors.Wrap(err, "parsing collection path")
errs = multierror.Append(errs, err)
logger.Ctx(ctx).Error(err)
return errs.ErrorOrNil()
}
items := streamedEnts.Items() items := streamedEnts.Items()
for { for {
@ -178,7 +165,7 @@ func getStreamItemFunc(
} }
// For now assuming that item IDs don't need escaping. // For now assuming that item IDs don't need escaping.
itemPath, err := itemPath.Append(e.UUID(), true) itemPath, err := streamedEnts.FullPath().Append(e.UUID(), true)
if err != nil { if err != nil {
err = errors.Wrap(err, "getting full item path") err = errors.Wrap(err, "getting full item path")
errs = multierror.Append(errs, err) errs = multierror.Append(errs, err)
@ -264,7 +251,11 @@ func inflateDirTree(
roots := make(map[string]*treeMap) roots := make(map[string]*treeMap)
for _, s := range collections { for _, s := range collections {
itemPath := s.FullPath() if s.FullPath() == nil {
return nil, errors.New("no identifier for collection")
}
itemPath := s.FullPath().Elements()
if len(itemPath) == 0 { if len(itemPath) == 0 {
return nil, errors.New("no identifier for collection") return nil, errors.New("no identifier for collection")

View File

@ -27,8 +27,8 @@ import (
const ( const (
testTenant = "a-tenant" testTenant = "a-tenant"
testUser = "user1" testUser = "user1"
testInboxDir = "inbox" testInboxDir = "Inbox"
testArchiveDir = "archive" testArchiveDir = "Archive"
testFileName = "file1" testFileName = "file1"
testFileName2 = "file2" testFileName2 = "file2"
testFileName3 = "file3" testFileName3 = "file3"
@ -75,9 +75,10 @@ func testForFiles(
for s := range c.Items() { for s := range c.Items() {
count++ count++
fullPath := stdpath.Join(append(c.FullPath(), s.UUID())...) fullPath, err := c.FullPath().Append(s.UUID(), true)
require.NoError(t, err)
expected, ok := expected[fullPath] expected, ok := expected[fullPath.String()]
require.True(t, ok, "unexpected file with path %q", fullPath) require.True(t, ok, "unexpected file with path %q", fullPath)
buf, err := ioutil.ReadAll(s.ToReader()) buf, err := ioutil.ReadAll(s.ToReader())
@ -214,6 +215,23 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
type KopiaUnitSuite struct { type KopiaUnitSuite struct {
suite.Suite suite.Suite
testPath path.Path
}
func (suite *KopiaUnitSuite) SetupSuite() {
tmp, err := path.FromDataLayerPath(
stdpath.Join(
testTenant,
path.ExchangeService.String(),
testUser,
path.EmailCategory.String(),
testInboxDir,
),
false,
)
require.NoError(suite.T(), err)
suite.testPath = tmp
} }
func TestKopiaUnitSuite(t *testing.T) { func TestKopiaUnitSuite(t *testing.T) {
@ -236,6 +254,18 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
user1 := testUser user1 := testUser
user2 := "user2" user2 := "user2"
p2, err := path.FromDataLayerPath(
stdpath.Join(
tenant,
service,
user2,
category,
testInboxDir,
),
false,
)
require.NoError(t, err)
expectedFileCount := map[string]int{ expectedFileCount := map[string]int{
user1: 5, user1: 5,
user2: 42, user2: 42,
@ -245,11 +275,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
collections := []data.Collection{ collections := []data.Collection{
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{tenant, service, user1, category, testInboxDir}, suite.testPath,
expectedFileCount[user1], expectedFileCount[user1],
), ),
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{tenant, service, user2, category, testInboxDir}, p2,
expectedFileCount[user2], expectedFileCount[user2],
), ),
} }
@ -301,6 +331,10 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() { func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
ctx := context.Background() ctx := context.Background()
subdir := "subfolder" subdir := "subfolder"
p2, err := suite.testPath.Append(subdir, false)
require.NoError(suite.T(), err)
// Test multiple orders of items because right now order can matter. Both // Test multiple orders of items because right now order can matter. Both
// orders result in a directory structure like: // orders result in a directory structure like:
// - a-tenant // - a-tenant
@ -319,11 +353,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
name: "SubdirFirst", name: "SubdirFirst",
layout: []data.Collection{ layout: []data.Collection{
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir, subdir}, p2,
5, 5,
), ),
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir}, suite.testPath,
42, 42,
), ),
}, },
@ -332,11 +366,11 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
name: "SubdirLast", name: "SubdirLast",
layout: []data.Collection{ layout: []data.Collection{
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir}, suite.testPath,
42, 42,
), ),
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{testTenant, service, testUser, category, testInboxDir, subdir}, p2,
5, 5,
), ),
}, },
@ -378,7 +412,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
} }
subDirs = append(subDirs, d) subDirs = append(subDirs, d)
assert.Equal(t, "subfolder", d.Name()) assert.Equal(t, subdir, d.Name())
} }
require.Len(t, subDirs, 1) require.Len(t, subDirs, 1)
@ -390,6 +424,14 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_MixedDirectory() {
} }
func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() { func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() {
p2, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory(
"tenant2",
"user2",
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
table := []struct { table := []struct {
name string name string
layout []data.Collection layout []data.Collection
@ -397,19 +439,25 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() {
{ {
"MultipleRoots", "MultipleRoots",
// Directory structure would look like: // Directory structure would look like:
// - user1 // - tenant1
// - emails // - exchange
// - 5 separate files // - user1
// - user2 // - emails
// - emails // - Inbox
// - 42 separate files // - 5 separate files
// - tenant2
// - exchange
// - user2
// - emails
// - Inbox
// - 42 separate files
[]data.Collection{ []data.Collection{
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{"user1", "emails"}, suite.testPath,
5, 5,
), ),
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{"user2", "emails"}, p2,
42, 42,
), ),
}, },
@ -457,6 +505,9 @@ type KopiaIntegrationSuite struct {
suite.Suite suite.Suite
w *Wrapper w *Wrapper
ctx context.Context ctx context.Context
testPath1 path.Path
testPath2 path.Path
} }
func TestKopiaIntegrationSuite(t *testing.T) { func TestKopiaIntegrationSuite(t *testing.T) {
@ -473,6 +524,26 @@ func TestKopiaIntegrationSuite(t *testing.T) {
func (suite *KopiaIntegrationSuite) SetupSuite() { func (suite *KopiaIntegrationSuite) SetupSuite() {
_, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...) _, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
tmp, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath1 = tmp
tmp, err = path.Builder{}.Append(testArchiveDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath2 = tmp
} }
func (suite *KopiaIntegrationSuite) SetupTest() { func (suite *KopiaIntegrationSuite) SetupTest() {
@ -494,11 +565,11 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
collections := []data.Collection{ collections := []data.Collection{
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user1", category, testInboxDir}, suite.testPath1,
5, 5,
), ),
mockconnector.NewMockExchangeCollection( mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user2", category, testInboxDir}, suite.testPath2,
42, 42,
), ),
} }
@ -506,7 +577,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
stats, rp, err := suite.w.BackupCollections(suite.ctx, collections) stats, rp, err := suite.w.BackupCollections(suite.ctx, collections)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, stats.TotalFileCount, 47) assert.Equal(t, stats.TotalFileCount, 47)
assert.Equal(t, stats.TotalDirectoryCount, 8) assert.Equal(t, stats.TotalDirectoryCount, 6)
assert.Equal(t, stats.IgnoredErrorCount, 0) assert.Equal(t, stats.IgnoredErrorCount, 0)
assert.Equal(t, stats.ErrorCount, 0) assert.Equal(t, stats.ErrorCount, 0)
assert.False(t, stats.Incomplete) assert.False(t, stats.Incomplete)
@ -524,26 +595,14 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
w := &Wrapper{k} w := &Wrapper{k}
tid := uuid.NewString() dc1 := mockconnector.NewMockExchangeCollection(suite.testPath1, 1)
p1 := []string{ dc2 := mockconnector.NewMockExchangeCollection(suite.testPath2, 1)
tid,
service,
"uid",
category,
"fid",
}
p2 := []string{
tid,
service,
"uid2",
category,
"fid",
}
dc1 := mockconnector.NewMockExchangeCollection(p1, 1)
dc2 := mockconnector.NewMockExchangeCollection(p2, 1)
fp1 := append(p1, dc1.Names[0]) fp1, err := suite.testPath1.Append(dc1.Names[0], true)
fp2 := append(p2, dc2.Names[0]) require.NoError(t, err)
fp2, err := suite.testPath2.Append(dc2.Names[0], true)
require.NoError(t, err)
stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}) stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2})
require.NoError(t, err) require.NoError(t, err)
@ -551,14 +610,19 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
require.NoError(t, k.Compression(ctx, "gzip")) require.NoError(t, k.Compression(ctx, "gzip"))
expected := map[string][]byte{ expected := map[string][]byte{
stdpath.Join(fp1...): dc1.Data[0], fp1.String(): dc1.Data[0],
stdpath.Join(fp2...): dc2.Data[0], fp2.String(): dc2.Data[0],
} }
result, err := w.RestoreMultipleItems( result, err := w.RestoreMultipleItems(
ctx, ctx,
string(stats.SnapshotID), string(stats.SnapshotID),
[][]string{fp1, fp2}) [][]string{
// TODO(ashmrtn): Remove when selectors passes paths to wrapper for
// restore.
fp1.Elements(),
fp2.Elements(),
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, 2, len(result)) assert.Equal(t, 2, len(result))
@ -568,29 +632,10 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
t := suite.T() t := suite.T()
tmpBuilder := path.Builder{}.Append(testInboxDir)
p1, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
tmpBuilder = path.Builder{}.Append(testArchiveDir)
p2, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
collections := []data.Collection{ collections := []data.Collection{
&kopiaDataCollection{ &kopiaDataCollection{
path: p1, path: suite.testPath1,
streams: []data.Stream{ streams: []data.Stream{
&mockconnector.MockExchangeData{ &mockconnector.MockExchangeData{
ID: testFileName, ID: testFileName,
@ -603,7 +648,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
}, },
}, },
&kopiaDataCollection{ &kopiaDataCollection{
path: p2, path: suite.testPath2,
streams: []data.Stream{ streams: []data.Stream{
&mockconnector.MockExchangeData{ &mockconnector.MockExchangeData{
ID: testFileName3, ID: testFileName3,
@ -644,6 +689,9 @@ type KopiaSimpleRepoIntegrationSuite struct {
inboxExpectedFiles map[string][]byte inboxExpectedFiles map[string][]byte
archiveExpectedFiles map[string][]byte archiveExpectedFiles map[string][]byte
allExpectedFiles map[string][]byte allExpectedFiles map[string][]byte
testPath1 path.Path
testPath2 path.Path
} }
func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) { func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) {
@ -660,6 +708,26 @@ func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) {
func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() {
_, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...) _, err := tester.GetRequiredEnvVars(tester.AWSStorageCredEnvs...)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
tmp, err := path.Builder{}.Append(testInboxDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath1 = tmp
tmp, err = path.Builder{}.Append(testArchiveDir).ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
suite.testPath2 = tmp
} }
func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
@ -669,29 +737,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
require.NoError(t, err) require.NoError(t, err)
suite.w = &Wrapper{c} suite.w = &Wrapper{c}
tmpBuilder := path.Builder{}.Append(testInboxDir)
p1, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
tmpBuilder = path.Builder{}.Append(testArchiveDir)
p2, err := tmpBuilder.ToDataLayerExchangePathForCategory(
testTenant,
testUser,
path.EmailCategory,
false,
)
require.NoError(t, err)
collections := []data.Collection{ collections := []data.Collection{
&kopiaDataCollection{ &kopiaDataCollection{
path: p1, path: suite.testPath1,
streams: []data.Stream{ streams: []data.Stream{
&mockconnector.MockExchangeData{ &mockconnector.MockExchangeData{
ID: testFileName, ID: testFileName,
@ -704,7 +753,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
}, },
}, },
&kopiaDataCollection{ &kopiaDataCollection{
path: p2, path: suite.testPath2,
streams: []data.Stream{ streams: []data.Stream{
&mockconnector.MockExchangeData{ &mockconnector.MockExchangeData{
ID: testFileName3, ID: testFileName3,
@ -773,7 +822,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupAndRestoreSingleItem() {
) )
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, c.FullPath(), testPath) assert.Equal(t, suite.testPath1, c.FullPath())
count := 0 count := 0
@ -839,39 +888,31 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() {
w := &Wrapper{k} w := &Wrapper{k}
tid := uuid.NewString() dc1 := mockconnector.NewMockExchangeCollection(suite.testPath1, 1)
p1 := []string{ dc2 := mockconnector.NewMockExchangeCollection(suite.testPath2, 1)
tid,
service,
"uid",
category,
"fid",
}
p2 := []string{
tid,
service,
"uid2",
category,
"fid",
}
dc1 := mockconnector.NewMockExchangeCollection(p1, 1)
dc2 := mockconnector.NewMockExchangeCollection(p2, 1)
fp1 := append(p1, dc1.Names[0]) fp1, err := suite.testPath1.Append(dc1.Names[0], true)
fp2 := append(p2, dc2.Names[0]) require.NoError(t, err)
fp2, err := suite.testPath2.Append(dc2.Names[0], true)
require.NoError(t, err)
stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}) stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2})
require.NoError(t, err) require.NoError(t, err)
expected := map[string][]byte{ expected := map[string][]byte{
stdpath.Join(fp1...): dc1.Data[0], fp1.String(): dc1.Data[0],
stdpath.Join(fp2...): dc2.Data[0], fp2.String(): dc2.Data[0],
} }
result, err := w.RestoreMultipleItems( result, err := w.RestoreMultipleItems(
ctx, ctx,
string(stats.SnapshotID), string(stats.SnapshotID),
[][]string{fp1, fp2}) [][]string{
// TODO(ashmrtn): Remove when selectors pass path to kopia restore.
fp1.Elements(),
fp2.Elements(),
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, 2, len(result)) assert.Equal(t, 2, len(result))
@ -919,34 +960,18 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors()
} }
} }
func (suite *KopiaIntegrationSuite) TestDeleteSnapshot() { func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() {
t := suite.T() t := suite.T()
dc1 := mockconnector.NewMockExchangeCollection( assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, string(suite.snapshotID)))
[]string{"a-tenant", service, "user1", category, testInboxDir},
5,
)
collections := []data.Collection{
dc1,
mockconnector.NewMockExchangeCollection(
[]string{"a-tenant", service, "user2", category, testInboxDir},
42,
),
}
bs, _, err := suite.w.BackupCollections(suite.ctx, collections)
require.NoError(t, err)
snapshotID := bs.SnapshotID
assert.NoError(t, suite.w.DeleteSnapshot(suite.ctx, snapshotID))
// assert the deletion worked // assert the deletion worked
itemPath := []string{"a-tenant", "user1", "emails", dc1.Names[0]} itemPath := append(testPath, testFileName)
_, err = suite.w.RestoreSingleItem(suite.ctx, snapshotID, itemPath) _, err := suite.w.RestoreSingleItem(suite.ctx, string(suite.snapshotID), itemPath)
assert.Error(t, err, "snapshot should be deleted") assert.Error(t, err, "snapshot should be deleted")
} }
func (suite *KopiaIntegrationSuite) TestDeleteSnapshot_BadIDs() { func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot_BadIDs() {
table := []struct { table := []struct {
name string name string
snapshotID string snapshotID string

View File

@ -74,6 +74,10 @@ type Path interface {
// If removing the right-most element would discard one of the required prefix // If removing the right-most element would discard one of the required prefix
// elements then an error is returned. // elements then an error is returned.
Dir() (Path, error) Dir() (Path, error)
// Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text
// elements in the future.
Elements() []string
// Append returns a new Path object with the given element added to the end of // Append returns a new Path object with the given element added to the end of
// the old Path if possible. If the old Path is an item Path then Append // the old Path if possible. If the old Path is an item Path then Append
// returns an error. // returns an error.