refactor onedrive test data creation (#3343)
<!-- PR description--> Currently we are duplicating the Onedrive test data creation code at two places- - in cli package - in test files. Moved the common code to connector package so that both cli and test files can use the same common code. #### Does this PR need a docs update or release note? - [ ] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🤖 Supportability/Tests #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * https://github.com/alcionai/corso/issues/3183 #### Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual
This commit is contained in:
neha_gupta
GitHub
parent
0bd524f86d
commit
694abfab68
@ -1,11 +1,8 @@
|
||||
package impl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@ -20,7 +17,6 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/connector"
|
||||
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
@ -83,7 +79,7 @@ func generateAndRestoreItems(
|
||||
}
|
||||
|
||||
collections := []collection{{
|
||||
pathElements: []string{destFldr},
|
||||
PathElements: []string{destFldr},
|
||||
category: cat,
|
||||
items: items,
|
||||
}}
|
||||
@ -160,7 +156,7 @@ type collection struct {
|
||||
// only contain elements after the prefix that corso uses for the path. For
|
||||
// example, a collection for the Inbox folder in exchange mail would just be
|
||||
// "Inbox".
|
||||
pathElements []string
|
||||
PathElements []string
|
||||
category path.CategoryType
|
||||
items []item
|
||||
}
|
||||
@ -180,7 +176,7 @@ func buildCollections(
|
||||
service,
|
||||
c.category,
|
||||
false,
|
||||
c.pathElements...)
|
||||
c.PathElements...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -198,45 +194,6 @@ func buildCollections(
|
||||
return collections, nil
|
||||
}
|
||||
|
||||
type permData struct {
|
||||
user string // user is only for older versions
|
||||
entityID string
|
||||
roles []string
|
||||
sharingMode metadata.SharingMode
|
||||
}
|
||||
|
||||
type itemData struct {
|
||||
name string
|
||||
data []byte
|
||||
perms permData
|
||||
}
|
||||
|
||||
type itemInfo struct {
|
||||
// lookupKey is a string that can be used to find this data from a set of
|
||||
// other data in the same collection. This key should be something that will
|
||||
// be the same before and after restoring the item in M365 and may not be
|
||||
// the M365 ID. When restoring items out of place, the item is assigned a
|
||||
// new ID making it unsuitable for a lookup key.
|
||||
lookupKey string
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type onedriveCollection struct {
|
||||
service path.ServiceType
|
||||
pathElements []string
|
||||
items []itemInfo
|
||||
aux []itemInfo
|
||||
backupVersion int
|
||||
}
|
||||
|
||||
type onedriveColInfo struct {
|
||||
pathElements []string
|
||||
perms permData
|
||||
files []itemData
|
||||
folders []itemData
|
||||
}
|
||||
|
||||
var (
|
||||
folderAName = "folder-a"
|
||||
folderBName = "b"
|
||||
@ -292,7 +249,7 @@ func generateAndRestoreDriveItems(
|
||||
}
|
||||
|
||||
var (
|
||||
cols []onedriveColInfo
|
||||
cols []connector.OnedriveColInfo
|
||||
|
||||
rootPath = []string{"drives", driveID, "root:"}
|
||||
folderAPath = []string{"drives", driveID, "root:", folderAName}
|
||||
@ -306,43 +263,43 @@ func generateAndRestoreDriveItems(
|
||||
)
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
col := []onedriveColInfo{
|
||||
col := []connector.OnedriveColInfo{
|
||||
// basic folder and file creation
|
||||
{
|
||||
pathElements: rootPath,
|
||||
files: []itemData{
|
||||
PathElements: rootPath,
|
||||
Files: []connector.ItemData{
|
||||
{
|
||||
name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
|
||||
Data: fileAData,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: fmt.Sprintf("file-2nd-count-%d-at-%s", i, currentTime),
|
||||
data: fileBData,
|
||||
Name: fmt.Sprintf("file-2nd-count-%d-at-%s", i, currentTime),
|
||||
Data: fileBData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []connector.ItemData{
|
||||
{
|
||||
name: folderBName,
|
||||
Name: folderBName,
|
||||
},
|
||||
{
|
||||
name: folderAName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Name: folderAName,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: folderCName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Name: folderCName,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -350,62 +307,62 @@ func generateAndRestoreDriveItems(
|
||||
{
|
||||
// a folder that has permissions with an item in the folder with
|
||||
// the different permissions.
|
||||
pathElements: folderAPath,
|
||||
files: []itemData{
|
||||
PathElements: folderAPath,
|
||||
Files: []connector.ItemData{
|
||||
{
|
||||
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
data: fileEData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
Data: fileEData,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
// a folder that has permissions with an item in the folder with
|
||||
// no permissions.
|
||||
pathElements: folderCPath,
|
||||
files: []itemData{
|
||||
PathElements: folderCPath,
|
||||
Files: []connector.ItemData{
|
||||
{
|
||||
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
data: fileAData,
|
||||
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
Data: fileAData,
|
||||
},
|
||||
},
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderBPath,
|
||||
files: []itemData{
|
||||
PathElements: folderBPath,
|
||||
Files: []connector.ItemData{
|
||||
{
|
||||
// restoring a file in a non-root folder that doesn't inherit
|
||||
// permissions.
|
||||
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
data: fileBData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
Data: fileBData,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []connector.ItemData{
|
||||
{
|
||||
name: folderAName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Name: folderAName,
|
||||
Perms: connector.PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -415,302 +372,40 @@ func generateAndRestoreDriveItems(
|
||||
cols = append(cols, col...)
|
||||
}
|
||||
|
||||
input := dataForInfo(service, cols, version.Backup)
|
||||
input, err := connector.DataForInfo(service, cols, version.Backup)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
collections := getCollections(
|
||||
service,
|
||||
tenantID,
|
||||
[]string{resourceOwner},
|
||||
input,
|
||||
version.Backup)
|
||||
// collections := getCollections(
|
||||
// service,
|
||||
// tenantID,
|
||||
// []string{resourceOwner},
|
||||
// input,
|
||||
// version.Backup)
|
||||
|
||||
opts := control.Options{
|
||||
RestorePermissions: true,
|
||||
ToggleFeatures: control.Toggles{},
|
||||
}
|
||||
|
||||
config := connector.ConfigInfo{
|
||||
Acct: acct,
|
||||
Opts: opts,
|
||||
Resource: connector.Users,
|
||||
Service: service,
|
||||
Tenant: tenantID,
|
||||
ResourceOwners: []string{resourceOwner},
|
||||
Dest: tester.DefaultTestRestoreDestination(""),
|
||||
}
|
||||
|
||||
_, _, collections, _, err := connector.GetCollectionsAndExpected(
|
||||
config,
|
||||
input,
|
||||
version.Backup)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return gc.ConsumeRestoreCollections(ctx, version.Backup, acct, sel, dest, opts, collections, errs)
|
||||
}
|
||||
|
||||
func getCollections(
|
||||
service path.ServiceType,
|
||||
tenant string,
|
||||
resourceOwners []string,
|
||||
testCollections []colInfo,
|
||||
backupVersion int,
|
||||
) []data.RestoreCollection {
|
||||
var collections []data.RestoreCollection
|
||||
|
||||
for _, owner := range resourceOwners {
|
||||
ownerCollections := collectionsForInfo(
|
||||
service,
|
||||
tenant,
|
||||
owner,
|
||||
testCollections,
|
||||
backupVersion,
|
||||
)
|
||||
|
||||
collections = append(collections, ownerCollections...)
|
||||
}
|
||||
|
||||
return collections
|
||||
}
|
||||
|
||||
type mockRestoreCollection struct {
|
||||
data.Collection
|
||||
auxItems map[string]data.Stream
|
||||
}
|
||||
|
||||
func (rc mockRestoreCollection) Fetch(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
) (data.Stream, error) {
|
||||
res := rc.auxItems[name]
|
||||
if res == nil {
|
||||
return nil, data.ErrNotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func collectionsForInfo(
|
||||
service path.ServiceType,
|
||||
tenant, user string,
|
||||
allInfo []colInfo,
|
||||
backupVersion int,
|
||||
) []data.RestoreCollection {
|
||||
collections := make([]data.RestoreCollection, 0, len(allInfo))
|
||||
|
||||
for _, info := range allInfo {
|
||||
pth := mustToDataLayerPath(
|
||||
service,
|
||||
tenant,
|
||||
user,
|
||||
info.category,
|
||||
info.pathElements,
|
||||
false)
|
||||
|
||||
mc := exchMock.NewCollection(pth, pth, len(info.items))
|
||||
|
||||
for i := 0; i < len(info.items); i++ {
|
||||
mc.Names[i] = info.items[i].name
|
||||
mc.Data[i] = info.items[i].data
|
||||
|
||||
// We do not count metadata files against item count
|
||||
if backupVersion > 0 && metadata.HasMetaSuffix(info.items[i].name) &&
|
||||
(service == path.OneDriveService || service == path.SharePointService) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
|
||||
|
||||
for _, aux := range info.auxItems {
|
||||
c.auxItems[aux.name] = &exchMock.Data{
|
||||
ID: aux.name,
|
||||
Reader: io.NopCloser(bytes.NewReader(aux.data)),
|
||||
}
|
||||
}
|
||||
|
||||
collections = append(collections, c)
|
||||
}
|
||||
|
||||
return collections
|
||||
}
|
||||
|
||||
func mustToDataLayerPath(
|
||||
service path.ServiceType,
|
||||
tenant, resourceOwner string,
|
||||
category path.CategoryType,
|
||||
elements []string,
|
||||
isItem bool,
|
||||
) path.Path {
|
||||
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
|
||||
if err != nil {
|
||||
fmt.Println("building path", clues.ToCore(err))
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
type colInfo struct {
|
||||
// Elements (in order) for the path representing this collection. Should
|
||||
// only contain elements after the prefix that corso uses for the path. For
|
||||
// example, a collection for the Inbox folder in exchange mail would just be
|
||||
// "Inbox".
|
||||
pathElements []string
|
||||
category path.CategoryType
|
||||
items []itemInfo
|
||||
// auxItems are items that can be retrieved with Fetch but won't be returned
|
||||
// by Items().
|
||||
auxItems []itemInfo
|
||||
}
|
||||
|
||||
func newOneDriveCollection(
|
||||
service path.ServiceType,
|
||||
pathElements []string,
|
||||
backupVersion int,
|
||||
) *onedriveCollection {
|
||||
return &onedriveCollection{
|
||||
service: service,
|
||||
pathElements: pathElements,
|
||||
backupVersion: backupVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func dataForInfo(
|
||||
service path.ServiceType,
|
||||
cols []onedriveColInfo,
|
||||
backupVersion int,
|
||||
) []colInfo {
|
||||
var res []colInfo
|
||||
|
||||
for _, c := range cols {
|
||||
onedriveCol := newOneDriveCollection(service, c.pathElements, backupVersion)
|
||||
|
||||
for _, f := range c.files {
|
||||
onedriveCol.withFile(f.name, f.data, f.perms)
|
||||
}
|
||||
|
||||
onedriveCol.withPermissions(c.perms)
|
||||
|
||||
res = append(res, onedriveCol.collection())
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (c onedriveCollection) collection() colInfo {
|
||||
cat := path.FilesCategory
|
||||
if c.service == path.SharePointService {
|
||||
cat = path.LibrariesCategory
|
||||
}
|
||||
|
||||
return colInfo{
|
||||
pathElements: c.pathElements,
|
||||
category: cat,
|
||||
items: c.items,
|
||||
auxItems: c.aux,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *onedriveCollection) withFile(name string, fileData []byte, perm permData) *onedriveCollection {
|
||||
c.items = append(c.items, onedriveItemWithData(
|
||||
name+metadata.DataFileSuffix,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData))
|
||||
|
||||
md := onedriveMetadata(
|
||||
name,
|
||||
name+metadata.MetaFileSuffix,
|
||||
name,
|
||||
perm,
|
||||
true)
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// withPermissions adds permissions to the folder represented by this
|
||||
// onedriveCollection.
|
||||
func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection {
|
||||
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
|
||||
return c
|
||||
}
|
||||
|
||||
name := c.pathElements[len(c.pathElements)-1]
|
||||
metaName := name
|
||||
|
||||
if c.backupVersion >= version.OneDrive5DirMetaNoName {
|
||||
// We switched to just .dirmeta for metadata file names.
|
||||
metaName = ""
|
||||
}
|
||||
|
||||
if name == "root:" {
|
||||
return c
|
||||
}
|
||||
|
||||
md := onedriveMetadata(
|
||||
name,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
perm,
|
||||
true)
|
||||
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
type oneDriveData struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Data []byte `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func onedriveItemWithData(
|
||||
name, lookupKey string,
|
||||
fileData []byte,
|
||||
) itemInfo {
|
||||
content := oneDriveData{
|
||||
FileName: lookupKey,
|
||||
Data: fileData,
|
||||
}
|
||||
|
||||
serialized, _ := json.Marshal(content)
|
||||
|
||||
return itemInfo{
|
||||
name: name,
|
||||
data: serialized,
|
||||
lookupKey: lookupKey,
|
||||
}
|
||||
}
|
||||
|
||||
func onedriveMetadata(
|
||||
fileName, itemID, lookupKey string,
|
||||
perm permData,
|
||||
permUseID bool,
|
||||
) itemInfo {
|
||||
meta := getMetadata(fileName, perm, permUseID)
|
||||
|
||||
metaJSON, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
fmt.Println("marshalling metadata", clues.ToCore(err))
|
||||
}
|
||||
|
||||
return itemInfo{
|
||||
name: itemID,
|
||||
data: metaJSON,
|
||||
lookupKey: lookupKey,
|
||||
}
|
||||
}
|
||||
|
||||
func getMetadata(fileName string, perm permData, permUseID bool) metadata.Metadata {
|
||||
if len(perm.user) == 0 || len(perm.roles) == 0 ||
|
||||
perm.sharingMode != metadata.SharingModeCustom {
|
||||
return metadata.Metadata{
|
||||
FileName: fileName,
|
||||
SharingMode: perm.sharingMode,
|
||||
}
|
||||
}
|
||||
|
||||
// In case of permissions, the id will usually be same for same
|
||||
// user/role combo unless deleted and readded, but we have to do
|
||||
// this as we only have two users of which one is already taken.
|
||||
id := uuid.NewString()
|
||||
uperm := metadata.Permission{ID: id, Roles: perm.roles}
|
||||
|
||||
if permUseID {
|
||||
uperm.EntityID = perm.entityID
|
||||
} else {
|
||||
uperm.Email = perm.user
|
||||
}
|
||||
|
||||
meta := metadata.Metadata{
|
||||
FileName: fileName,
|
||||
Permissions: []metadata.Permission{uperm},
|
||||
}
|
||||
|
||||
return meta
|
||||
}
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package connector
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
@ -17,33 +16,16 @@ import (
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
func mustToDataLayerPath(
|
||||
t *testing.T,
|
||||
service path.ServiceType,
|
||||
tenant, resourceOwner string,
|
||||
category path.CategoryType,
|
||||
elements []string,
|
||||
isItem bool,
|
||||
) path.Path {
|
||||
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func testElementsMatch[T any](
|
||||
t *testing.T,
|
||||
expected []T,
|
||||
@ -116,52 +98,17 @@ func testElementsMatch[T any](
|
||||
unexpected)
|
||||
}
|
||||
|
||||
type configInfo struct {
|
||||
acct account.Account
|
||||
opts control.Options
|
||||
resource Resource
|
||||
service path.ServiceType
|
||||
tenant string
|
||||
resourceOwners []string
|
||||
dest control.RestoreDestination
|
||||
}
|
||||
|
||||
type itemInfo struct {
|
||||
// lookupKey is a string that can be used to find this data from a set of
|
||||
// other data in the same collection. This key should be something that will
|
||||
// be the same before and after restoring the item in M365 and may not be
|
||||
// the M365 ID. When restoring items out of place, the item is assigned a
|
||||
// new ID making it unsuitable for a lookup key.
|
||||
lookupKey string
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type colInfo struct {
|
||||
// Elements (in order) for the path representing this collection. Should
|
||||
// only contain elements after the prefix that corso uses for the path. For
|
||||
// example, a collection for the Inbox folder in exchange mail would just be
|
||||
// "Inbox".
|
||||
pathElements []string
|
||||
category path.CategoryType
|
||||
items []itemInfo
|
||||
// auxItems are items that can be retrieved with Fetch but won't be returned
|
||||
// by Items(). These files do not directly participate in comparisosn at the
|
||||
// end of a test.
|
||||
auxItems []itemInfo
|
||||
}
|
||||
|
||||
type restoreBackupInfo struct {
|
||||
name string
|
||||
service path.ServiceType
|
||||
collections []colInfo
|
||||
collections []ColInfo
|
||||
resource Resource
|
||||
}
|
||||
|
||||
type restoreBackupInfoMultiVersion struct {
|
||||
service path.ServiceType
|
||||
collectionsLatest []colInfo
|
||||
collectionsPrevious []colInfo
|
||||
collectionsLatest []ColInfo
|
||||
collectionsPrevious []ColInfo
|
||||
resource Resource
|
||||
backupVersion int
|
||||
}
|
||||
@ -734,7 +681,7 @@ func compareDriveItem(
|
||||
t *testing.T,
|
||||
expected map[string][]byte,
|
||||
item data.Stream,
|
||||
config configInfo,
|
||||
config ConfigInfo,
|
||||
rootDir bool,
|
||||
) bool {
|
||||
// Skip Drive permissions in the folder that used to be the root. We don't
|
||||
@ -814,7 +761,7 @@ func compareDriveItem(
|
||||
assert.Equal(t, expectedMeta.FileName, itemMeta.FileName)
|
||||
}
|
||||
|
||||
if !config.opts.RestorePermissions {
|
||||
if !config.Opts.RestorePermissions {
|
||||
assert.Equal(t, 0, len(itemMeta.Permissions))
|
||||
return true
|
||||
}
|
||||
@ -835,7 +782,7 @@ func compareDriveItem(
|
||||
// sharepoint retrieves a superset of permissions
|
||||
// (all site admins, site groups, built in by default)
|
||||
// relative to the permissions changed by the test.
|
||||
config.service == path.SharePointService,
|
||||
config.Service == path.SharePointService,
|
||||
permissionEqual)
|
||||
|
||||
return true
|
||||
@ -877,7 +824,7 @@ func compareItem(
|
||||
service path.ServiceType,
|
||||
category path.CategoryType,
|
||||
item data.Stream,
|
||||
config configInfo,
|
||||
config ConfigInfo,
|
||||
rootDir bool,
|
||||
) bool {
|
||||
if mt, ok := item.(data.StreamModTime); ok {
|
||||
@ -971,7 +918,7 @@ func checkCollections(
|
||||
expectedItems int,
|
||||
expected map[string]map[string][]byte,
|
||||
got []data.BackupCollection,
|
||||
config configInfo,
|
||||
config ConfigInfo,
|
||||
) int {
|
||||
collectionsWithItems := []data.BackupCollection{}
|
||||
|
||||
@ -985,7 +932,7 @@ func checkCollections(
|
||||
category = returned.FullPath().Category()
|
||||
expectedColData = expected[returned.FullPath().String()]
|
||||
folders = returned.FullPath().Elements()
|
||||
rootDir = folders[len(folders)-1] == config.dest.ContainerName
|
||||
rootDir = folders[len(folders)-1] == config.Dest.ContainerName
|
||||
)
|
||||
|
||||
// Need to iterate through all items even if we don't expect to find a match
|
||||
@ -1166,127 +1113,6 @@ func backupSelectorForExpected(
|
||||
return selectors.Selector{}
|
||||
}
|
||||
|
||||
// backupOutputPathFromRestore returns a path.Path denoting the location in
|
||||
// kopia the data will be placed at. The location is a data-type specific
|
||||
// combination of the location the data was recently restored to and where the
|
||||
// data was originally in the hierarchy.
|
||||
func backupOutputPathFromRestore(
|
||||
t *testing.T,
|
||||
restoreDest control.RestoreDestination,
|
||||
inputPath path.Path,
|
||||
) path.Path {
|
||||
base := []string{restoreDest.ContainerName}
|
||||
|
||||
// OneDrive has leading information like the drive ID.
|
||||
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService {
|
||||
folders := inputPath.Folders()
|
||||
base = append(append([]string{}, folders[:3]...), restoreDest.ContainerName)
|
||||
|
||||
if len(folders) > 3 {
|
||||
base = append(base, folders[3:]...)
|
||||
}
|
||||
}
|
||||
|
||||
if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory {
|
||||
base = append(base, inputPath.Folders()...)
|
||||
}
|
||||
|
||||
return mustToDataLayerPath(
|
||||
t,
|
||||
inputPath.Service(),
|
||||
inputPath.Tenant(),
|
||||
inputPath.ResourceOwner(),
|
||||
inputPath.Category(),
|
||||
base,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
// TODO(ashmrtn): Make this an actual mock class that can be used in other
|
||||
// packages.
|
||||
type mockRestoreCollection struct {
|
||||
data.Collection
|
||||
auxItems map[string]data.Stream
|
||||
}
|
||||
|
||||
func (rc mockRestoreCollection) Fetch(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
) (data.Stream, error) {
|
||||
res := rc.auxItems[name]
|
||||
if res == nil {
|
||||
return nil, data.ErrNotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func collectionsForInfo(
|
||||
t *testing.T,
|
||||
service path.ServiceType,
|
||||
tenant, user string,
|
||||
dest control.RestoreDestination,
|
||||
allInfo []colInfo,
|
||||
backupVersion int,
|
||||
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
|
||||
var (
|
||||
collections = make([]data.RestoreCollection, 0, len(allInfo))
|
||||
expectedData = make(map[string]map[string][]byte, len(allInfo))
|
||||
totalItems = 0
|
||||
kopiaEntries = 0
|
||||
)
|
||||
|
||||
for _, info := range allInfo {
|
||||
pth := mustToDataLayerPath(
|
||||
t,
|
||||
service,
|
||||
tenant,
|
||||
user,
|
||||
info.category,
|
||||
info.pathElements,
|
||||
false)
|
||||
|
||||
mc := exchMock.NewCollection(pth, pth, len(info.items))
|
||||
baseDestPath := backupOutputPathFromRestore(t, dest, pth)
|
||||
|
||||
baseExpected := expectedData[baseDestPath.String()]
|
||||
if len(baseExpected) == 0 {
|
||||
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.items))
|
||||
baseExpected = expectedData[baseDestPath.String()]
|
||||
}
|
||||
|
||||
for i := 0; i < len(info.items); i++ {
|
||||
mc.Names[i] = info.items[i].name
|
||||
mc.Data[i] = info.items[i].data
|
||||
|
||||
baseExpected[info.items[i].lookupKey] = info.items[i].data
|
||||
|
||||
// We do not count metadata files against item count
|
||||
if backupVersion > 0 &&
|
||||
(service == path.OneDriveService || service == path.SharePointService) &&
|
||||
metadata.HasMetaSuffix(info.items[i].name) {
|
||||
continue
|
||||
}
|
||||
|
||||
totalItems++
|
||||
}
|
||||
|
||||
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
|
||||
|
||||
for _, aux := range info.auxItems {
|
||||
c.auxItems[aux.name] = &exchMock.Data{
|
||||
ID: aux.name,
|
||||
Reader: io.NopCloser(bytes.NewReader(aux.data)),
|
||||
}
|
||||
}
|
||||
|
||||
collections = append(collections, c)
|
||||
kopiaEntries += len(info.items)
|
||||
}
|
||||
|
||||
return totalItems, kopiaEntries, collections, expectedData
|
||||
}
|
||||
|
||||
func getSelectorWith(
|
||||
t *testing.T,
|
||||
service path.ServiceType,
|
||||
|
||||
@ -2,13 +2,11 @@ package connector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/google/uuid"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -25,86 +23,6 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
// For any version post this(inclusive), we expect to be using IDs for
|
||||
// permission instead of email
|
||||
const versionPermissionSwitchedToID = version.OneDrive4DirIncludesPermissions
|
||||
|
||||
func getMetadata(fileName string, perm permData, permUseID bool) metadata.Metadata {
|
||||
if len(perm.user) == 0 || len(perm.roles) == 0 ||
|
||||
perm.sharingMode != metadata.SharingModeCustom {
|
||||
return metadata.Metadata{
|
||||
FileName: fileName,
|
||||
SharingMode: perm.sharingMode,
|
||||
}
|
||||
}
|
||||
|
||||
// In case of permissions, the id will usually be same for same
|
||||
// user/role combo unless deleted and readded, but we have to do
|
||||
// this as we only have two users of which one is already taken.
|
||||
id := uuid.NewString()
|
||||
uperm := metadata.Permission{ID: id, Roles: perm.roles}
|
||||
|
||||
if permUseID {
|
||||
uperm.EntityID = perm.entityID
|
||||
} else {
|
||||
uperm.Email = perm.user
|
||||
}
|
||||
|
||||
testMeta := metadata.Metadata{
|
||||
FileName: fileName,
|
||||
Permissions: []metadata.Permission{uperm},
|
||||
}
|
||||
|
||||
return testMeta
|
||||
}
|
||||
|
||||
type testOneDriveData struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Data []byte `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func onedriveItemWithData(
|
||||
t *testing.T,
|
||||
name, lookupKey string,
|
||||
fileData []byte,
|
||||
) itemInfo {
|
||||
t.Helper()
|
||||
|
||||
content := testOneDriveData{
|
||||
FileName: lookupKey,
|
||||
Data: fileData,
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(content)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
return itemInfo{
|
||||
name: name,
|
||||
data: serialized,
|
||||
lookupKey: lookupKey,
|
||||
}
|
||||
}
|
||||
|
||||
func onedriveMetadata(
|
||||
t *testing.T,
|
||||
fileName, itemID, lookupKey string,
|
||||
perm permData,
|
||||
permUseID bool,
|
||||
) itemInfo {
|
||||
t.Helper()
|
||||
|
||||
testMeta := getMetadata(fileName, perm, permUseID)
|
||||
|
||||
testMetaJSON, err := json.Marshal(testMeta)
|
||||
require.NoError(t, err, "marshalling metadata", clues.ToCore(err))
|
||||
|
||||
return itemInfo{
|
||||
name: itemID,
|
||||
data: testMetaJSON,
|
||||
lookupKey: lookupKey,
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
fileName = "test-file.txt"
|
||||
folderAName = "folder-a"
|
||||
@ -122,204 +40,6 @@ var (
|
||||
readPerm = []string{"read"}
|
||||
)
|
||||
|
||||
func newOneDriveCollection(
|
||||
t *testing.T,
|
||||
service path.ServiceType,
|
||||
pathElements []string,
|
||||
backupVersion int,
|
||||
) *onedriveCollection {
|
||||
return &onedriveCollection{
|
||||
service: service,
|
||||
pathElements: pathElements,
|
||||
backupVersion: backupVersion,
|
||||
t: t,
|
||||
}
|
||||
}
|
||||
|
||||
type onedriveCollection struct {
|
||||
service path.ServiceType
|
||||
pathElements []string
|
||||
items []itemInfo
|
||||
aux []itemInfo
|
||||
backupVersion int
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (c onedriveCollection) collection() colInfo {
|
||||
cat := path.FilesCategory
|
||||
if c.service == path.SharePointService {
|
||||
cat = path.LibrariesCategory
|
||||
}
|
||||
|
||||
return colInfo{
|
||||
pathElements: c.pathElements,
|
||||
category: cat,
|
||||
items: c.items,
|
||||
auxItems: c.aux,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *onedriveCollection) withFile(name string, fileData []byte, perm permData) *onedriveCollection {
|
||||
switch c.backupVersion {
|
||||
case 0:
|
||||
// Lookups will occur using the most recent version of things so we need
|
||||
// the embedded file name to match that.
|
||||
c.items = append(c.items, onedriveItemWithData(
|
||||
c.t,
|
||||
name,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData))
|
||||
|
||||
// v1-5, early metadata design
|
||||
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker,
|
||||
version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName:
|
||||
c.items = append(c.items, onedriveItemWithData(
|
||||
c.t,
|
||||
name+metadata.DataFileSuffix,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData))
|
||||
|
||||
md := onedriveMetadata(
|
||||
c.t,
|
||||
"",
|
||||
name+metadata.MetaFileSuffix,
|
||||
name+metadata.MetaFileSuffix,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
// v6+ current metadata design
|
||||
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
|
||||
c.items = append(c.items, onedriveItemWithData(
|
||||
c.t,
|
||||
name+metadata.DataFileSuffix,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData))
|
||||
|
||||
md := onedriveMetadata(
|
||||
c.t,
|
||||
name,
|
||||
name+metadata.MetaFileSuffix,
|
||||
name,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
default:
|
||||
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *onedriveCollection) withFolder(name string, perm permData) *onedriveCollection {
|
||||
switch c.backupVersion {
|
||||
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
|
||||
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
|
||||
return c
|
||||
|
||||
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
|
||||
c.items = append(
|
||||
c.items,
|
||||
onedriveMetadata(
|
||||
c.t,
|
||||
"",
|
||||
name+metadata.DirMetaFileSuffix,
|
||||
name+metadata.DirMetaFileSuffix,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID))
|
||||
|
||||
default:
|
||||
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// withPermissions adds permissions to the folder represented by this
|
||||
// onedriveCollection.
|
||||
func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection {
|
||||
// These versions didn't store permissions for the folder or didn't store them
|
||||
// in the folder's collection.
|
||||
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
|
||||
return c
|
||||
}
|
||||
|
||||
name := c.pathElements[len(c.pathElements)-1]
|
||||
metaName := name
|
||||
|
||||
if c.backupVersion >= version.OneDrive5DirMetaNoName {
|
||||
// We switched to just .dirmeta for metadata file names.
|
||||
metaName = ""
|
||||
}
|
||||
|
||||
if name == odConsts.RootPathDir {
|
||||
return c
|
||||
}
|
||||
|
||||
md := onedriveMetadata(
|
||||
c.t,
|
||||
name,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
type permData struct {
|
||||
user string // user is only for older versions
|
||||
entityID string
|
||||
roles []string
|
||||
sharingMode metadata.SharingMode
|
||||
}
|
||||
|
||||
type itemData struct {
|
||||
name string
|
||||
data []byte
|
||||
perms permData
|
||||
}
|
||||
|
||||
type driveColInfo struct {
|
||||
pathElements []string
|
||||
perms permData
|
||||
files []itemData
|
||||
folders []itemData
|
||||
}
|
||||
|
||||
func testDataForInfo(
|
||||
t *testing.T,
|
||||
service path.ServiceType,
|
||||
cols []driveColInfo,
|
||||
backupVersion int,
|
||||
) []colInfo {
|
||||
var res []colInfo
|
||||
|
||||
for _, c := range cols {
|
||||
onedriveCol := newOneDriveCollection(t, service, c.pathElements, backupVersion)
|
||||
|
||||
for _, f := range c.files {
|
||||
onedriveCol.withFile(f.name, f.data, f.perms)
|
||||
}
|
||||
|
||||
for _, d := range c.folders {
|
||||
onedriveCol.withFolder(d.name, d.perms)
|
||||
}
|
||||
|
||||
onedriveCol.withPermissions(c.perms)
|
||||
|
||||
res = append(res, onedriveCol.collection())
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func mustGetDefaultDriveID(
|
||||
t *testing.T,
|
||||
ctx context.Context, //revive:disable-line:context-as-argument
|
||||
@ -675,78 +395,80 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
|
||||
folderBName,
|
||||
}
|
||||
|
||||
cols := []driveColInfo{
|
||||
cols := []OnedriveColInfo{
|
||||
{
|
||||
pathElements: rootPath,
|
||||
files: []itemData{
|
||||
PathElements: rootPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileAData,
|
||||
Name: fileName,
|
||||
Data: fileAData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderAName,
|
||||
Name: folderAName,
|
||||
},
|
||||
{
|
||||
name: folderBName,
|
||||
Name: folderBName,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderAPath,
|
||||
files: []itemData{
|
||||
PathElements: folderAPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileBData,
|
||||
Name: fileName,
|
||||
Data: fileBData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderBName,
|
||||
Name: folderBName,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: subfolderBPath,
|
||||
files: []itemData{
|
||||
PathElements: subfolderBPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileCData,
|
||||
Name: fileName,
|
||||
Data: fileCData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderAName,
|
||||
Name: folderAName,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: subfolderAPath,
|
||||
files: []itemData{
|
||||
PathElements: subfolderAPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileDData,
|
||||
Name: fileName,
|
||||
Data: fileDData,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderBPath,
|
||||
files: []itemData{
|
||||
PathElements: folderBPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileEData,
|
||||
Name: fileName,
|
||||
Data: fileEData,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
|
||||
expected, err := DataForInfo(suite.BackupService(), cols, version.Backup)
|
||||
require.NoError(suite.T(), err)
|
||||
|
||||
for vn := startVersion; vn <= version.Backup; vn++ {
|
||||
suite.Run(fmt.Sprintf("Version%d", vn), func() {
|
||||
t := suite.T()
|
||||
input := testDataForInfo(t, suite.BackupService(), cols, vn)
|
||||
input, err := DataForInfo(suite.BackupService(), cols, vn)
|
||||
require.NoError(suite.T(), err)
|
||||
|
||||
testData := restoreBackupInfoMultiVersion{
|
||||
service: suite.BackupService(),
|
||||
@ -819,71 +541,71 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
|
||||
folderCName,
|
||||
}
|
||||
|
||||
cols := []driveColInfo{
|
||||
cols := []OnedriveColInfo{
|
||||
{
|
||||
pathElements: rootPath,
|
||||
files: []itemData{
|
||||
PathElements: rootPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
// Test restoring a file that doesn't inherit permissions.
|
||||
name: fileName,
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fileName,
|
||||
Data: fileAData,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
// Test restoring a file that doesn't inherit permissions and has
|
||||
// no permissions.
|
||||
name: fileName2,
|
||||
data: fileBData,
|
||||
Name: fileName2,
|
||||
Data: fileBData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderBName,
|
||||
Name: folderBName,
|
||||
},
|
||||
{
|
||||
name: folderAName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Name: folderAName,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: folderCName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Name: folderCName,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderBPath,
|
||||
files: []itemData{
|
||||
PathElements: folderBPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
// Test restoring a file in a non-root folder that doesn't inherit
|
||||
// permissions.
|
||||
name: fileName,
|
||||
data: fileBData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fileName,
|
||||
Data: fileBData,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderAName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Name: folderAName,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -905,52 +627,53 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
|
||||
// },
|
||||
// },
|
||||
// },
|
||||
// perms: permData{
|
||||
// user: secondaryUserName,
|
||||
// entityID: secondaryUserID,
|
||||
// roles: readPerm,
|
||||
// Perms: PermData{
|
||||
// User: secondaryUserName,
|
||||
// EntityID: secondaryUserID,
|
||||
// Roles: readPerm,
|
||||
// },
|
||||
// },
|
||||
{
|
||||
// Tests a folder that has permissions with an item in the folder with
|
||||
// the different permissions.
|
||||
pathElements: folderAPath,
|
||||
files: []itemData{
|
||||
PathElements: folderAPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileEData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fileName,
|
||||
Data: fileEData,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
// Tests a folder that has permissions with an item in the folder with
|
||||
// no permissions.
|
||||
pathElements: folderCPath,
|
||||
files: []itemData{
|
||||
PathElements: folderCPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileAData,
|
||||
Name: fileName,
|
||||
Data: fileAData,
|
||||
},
|
||||
},
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
|
||||
expected, err := DataForInfo(suite.BackupService(), cols, version.Backup)
|
||||
require.NoError(suite.T(), err)
|
||||
bss := suite.BackupService().String()
|
||||
|
||||
for vn := startVersion; vn <= version.Backup; vn++ {
|
||||
@ -959,7 +682,8 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
|
||||
// Ideally this can always be true or false and still
|
||||
// work, but limiting older versions to use emails so as
|
||||
// to validate that flow as well.
|
||||
input := testDataForInfo(t, suite.BackupService(), cols, vn)
|
||||
input, err := DataForInfo(suite.BackupService(), cols, vn)
|
||||
require.NoError(suite.T(), err)
|
||||
|
||||
testData := restoreBackupInfoMultiVersion{
|
||||
service: suite.BackupService(),
|
||||
@ -997,51 +721,53 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
|
||||
suite.Service(),
|
||||
suite.BackupResourceOwner())
|
||||
|
||||
inputCols := []driveColInfo{
|
||||
inputCols := []OnedriveColInfo{
|
||||
{
|
||||
pathElements: []string{
|
||||
PathElements: []string{
|
||||
odConsts.DrivesPathDir,
|
||||
driveID,
|
||||
odConsts.RootPathDir,
|
||||
},
|
||||
files: []itemData{
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
Name: fileName,
|
||||
Data: fileAData,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expectedCols := []driveColInfo{
|
||||
expectedCols := []OnedriveColInfo{
|
||||
{
|
||||
pathElements: []string{
|
||||
PathElements: []string{
|
||||
odConsts.DrivesPathDir,
|
||||
driveID,
|
||||
odConsts.RootPathDir,
|
||||
},
|
||||
files: []itemData{
|
||||
Files: []ItemData{
|
||||
{
|
||||
// No permissions on the output since they weren't restored.
|
||||
name: fileName,
|
||||
data: fileAData,
|
||||
Name: fileName,
|
||||
Data: fileAData,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expected := testDataForInfo(suite.T(), suite.BackupService(), expectedCols, version.Backup)
|
||||
expected, err := DataForInfo(suite.BackupService(), expectedCols, version.Backup)
|
||||
require.NoError(suite.T(), err)
|
||||
bss := suite.BackupService().String()
|
||||
|
||||
for vn := startVersion; vn <= version.Backup; vn++ {
|
||||
suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() {
|
||||
t := suite.T()
|
||||
input := testDataForInfo(t, suite.BackupService(), inputCols, vn)
|
||||
input, err := DataForInfo(suite.BackupService(), inputCols, vn)
|
||||
require.NoError(suite.T(), err)
|
||||
|
||||
testData := restoreBackupInfoMultiVersion{
|
||||
service: suite.BackupService(),
|
||||
@ -1119,29 +845,29 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
|
||||
folderCName,
|
||||
}
|
||||
|
||||
fileSet := []itemData{
|
||||
fileSet := []ItemData{
|
||||
{
|
||||
name: "file-custom",
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
sharingMode: metadata.SharingModeCustom,
|
||||
Name: "file-custom",
|
||||
Data: fileAData,
|
||||
Perms: PermData{
|
||||
User: secondaryUserName,
|
||||
EntityID: secondaryUserID,
|
||||
Roles: writePerm,
|
||||
SharingMode: metadata.SharingModeCustom,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "file-inherited",
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
sharingMode: metadata.SharingModeInherited,
|
||||
Name: "file-inherited",
|
||||
Data: fileAData,
|
||||
Perms: PermData{
|
||||
SharingMode: metadata.SharingModeInherited,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "file-empty",
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
sharingMode: metadata.SharingModeCustom,
|
||||
Name: "file-empty",
|
||||
Data: fileAData,
|
||||
Perms: PermData{
|
||||
SharingMode: metadata.SharingModeCustom,
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -1164,55 +890,56 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
|
||||
// - inherted-permission-file
|
||||
// - empty-permission-file (empty/empty might have interesting behavior)
|
||||
|
||||
cols := []driveColInfo{
|
||||
cols := []OnedriveColInfo{
|
||||
{
|
||||
pathElements: rootPath,
|
||||
files: []itemData{},
|
||||
folders: []itemData{
|
||||
{name: folderAName},
|
||||
PathElements: rootPath,
|
||||
Files: []ItemData{},
|
||||
Folders: []ItemData{
|
||||
{Name: folderAName},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderAPath,
|
||||
files: fileSet,
|
||||
folders: []itemData{
|
||||
{name: folderAName},
|
||||
{name: folderBName},
|
||||
{name: folderCName},
|
||||
PathElements: folderAPath,
|
||||
Files: fileSet,
|
||||
Folders: []ItemData{
|
||||
{Name: folderAName},
|
||||
{Name: folderBName},
|
||||
{Name: folderCName},
|
||||
},
|
||||
perms: permData{
|
||||
user: tertiaryUserName,
|
||||
entityID: tertiaryUserID,
|
||||
roles: readPerm,
|
||||
Perms: PermData{
|
||||
User: tertiaryUserName,
|
||||
EntityID: tertiaryUserID,
|
||||
Roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: subfolderAAPath,
|
||||
files: fileSet,
|
||||
perms: permData{
|
||||
user: tertiaryUserName,
|
||||
entityID: tertiaryUserID,
|
||||
roles: writePerm,
|
||||
sharingMode: metadata.SharingModeCustom,
|
||||
PathElements: subfolderAAPath,
|
||||
Files: fileSet,
|
||||
Perms: PermData{
|
||||
User: tertiaryUserName,
|
||||
EntityID: tertiaryUserID,
|
||||
Roles: writePerm,
|
||||
SharingMode: metadata.SharingModeCustom,
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: subfolderABPath,
|
||||
files: fileSet,
|
||||
perms: permData{
|
||||
sharingMode: metadata.SharingModeInherited,
|
||||
PathElements: subfolderABPath,
|
||||
Files: fileSet,
|
||||
Perms: PermData{
|
||||
SharingMode: metadata.SharingModeInherited,
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: subfolderACPath,
|
||||
files: fileSet,
|
||||
perms: permData{
|
||||
sharingMode: metadata.SharingModeCustom,
|
||||
PathElements: subfolderACPath,
|
||||
Files: fileSet,
|
||||
Perms: PermData{
|
||||
SharingMode: metadata.SharingModeCustom,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
|
||||
expected, err := DataForInfo(suite.BackupService(), cols, version.Backup)
|
||||
require.NoError(suite.T(), err)
|
||||
bss := suite.BackupService().String()
|
||||
|
||||
for vn := startVersion; vn <= version.Backup; vn++ {
|
||||
@ -1221,7 +948,8 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
|
||||
// Ideally this can always be true or false and still
|
||||
// work, but limiting older versions to use emails so as
|
||||
// to validate that flow as well.
|
||||
input := testDataForInfo(t, suite.BackupService(), cols, vn)
|
||||
input, err := DataForInfo(suite.BackupService(), cols, vn)
|
||||
require.NoError(suite.T(), err)
|
||||
|
||||
testData := restoreBackupInfoMultiVersion{
|
||||
service: suite.BackupService(),
|
||||
@ -1279,61 +1007,63 @@ func testRestoreFolderNamedFolderRegression(
|
||||
folderBName,
|
||||
}
|
||||
|
||||
cols := []driveColInfo{
|
||||
cols := []OnedriveColInfo{
|
||||
{
|
||||
pathElements: rootPath,
|
||||
files: []itemData{
|
||||
PathElements: rootPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileAData,
|
||||
Name: fileName,
|
||||
Data: fileAData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderNamedFolder,
|
||||
Name: folderNamedFolder,
|
||||
},
|
||||
{
|
||||
name: folderBName,
|
||||
Name: folderBName,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderFolderPath,
|
||||
files: []itemData{
|
||||
PathElements: folderFolderPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileBData,
|
||||
Name: fileName,
|
||||
Data: fileBData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderBName,
|
||||
Name: folderBName,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: subfolderPath,
|
||||
files: []itemData{
|
||||
PathElements: subfolderPath,
|
||||
Files: []ItemData{
|
||||
{
|
||||
name: fileName,
|
||||
data: fileCData,
|
||||
Name: fileName,
|
||||
Data: fileCData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
Folders: []ItemData{
|
||||
{
|
||||
name: folderNamedFolder,
|
||||
Name: folderNamedFolder,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
|
||||
expected, err := DataForInfo(suite.BackupService(), cols, version.Backup)
|
||||
require.NoError(suite.T(), err)
|
||||
bss := suite.BackupService().String()
|
||||
|
||||
for vn := startVersion; vn <= version.Backup; vn++ {
|
||||
suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() {
|
||||
t := suite.T()
|
||||
input := testDataForInfo(t, suite.BackupService(), cols, vn)
|
||||
input, err := DataForInfo(suite.BackupService(), cols, vn)
|
||||
require.NoError(suite.T(), err)
|
||||
|
||||
testData := restoreBackupInfoMultiVersion{
|
||||
service: suite.BackupService(),
|
||||
|
||||
358
src/internal/connector/graph_connector_onedrive_test_helper.go
Normal file
358
src/internal/connector/graph_connector_onedrive_test_helper.go
Normal file
@ -0,0 +1,358 @@
|
||||
package connector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
// For any version post this(inclusive), we expect to be using IDs for
|
||||
// permission instead of email
|
||||
const versionPermissionSwitchedToID = version.OneDrive4DirIncludesPermissions
|
||||
|
||||
func getMetadata(fileName string, perm PermData, permUseID bool) metadata.Metadata {
|
||||
if len(perm.User) == 0 || len(perm.Roles) == 0 ||
|
||||
perm.SharingMode != metadata.SharingModeCustom {
|
||||
return metadata.Metadata{
|
||||
FileName: fileName,
|
||||
SharingMode: perm.SharingMode,
|
||||
}
|
||||
}
|
||||
|
||||
// In case of permissions, the id will usually be same for same
|
||||
// user/role combo unless deleted and readded, but we have to do
|
||||
// this as we only have two users of which one is already taken.
|
||||
id := uuid.NewString()
|
||||
uperm := metadata.Permission{ID: id, Roles: perm.Roles}
|
||||
|
||||
if permUseID {
|
||||
uperm.EntityID = perm.EntityID
|
||||
} else {
|
||||
uperm.Email = perm.User
|
||||
}
|
||||
|
||||
testMeta := metadata.Metadata{
|
||||
FileName: fileName,
|
||||
Permissions: []metadata.Permission{uperm},
|
||||
}
|
||||
|
||||
return testMeta
|
||||
}
|
||||
|
||||
type PermData struct {
|
||||
User string // user is only for older versions
|
||||
EntityID string
|
||||
Roles []string
|
||||
SharingMode metadata.SharingMode
|
||||
}
|
||||
|
||||
type ItemData struct {
|
||||
Name string
|
||||
Data []byte
|
||||
Perms PermData
|
||||
}
|
||||
|
||||
type OnedriveColInfo struct {
|
||||
PathElements []string
|
||||
Perms PermData
|
||||
Files []ItemData
|
||||
Folders []ItemData
|
||||
}
|
||||
|
||||
type onedriveCollection struct {
|
||||
service path.ServiceType
|
||||
PathElements []string
|
||||
items []ItemInfo
|
||||
aux []ItemInfo
|
||||
backupVersion int
|
||||
}
|
||||
|
||||
func (c onedriveCollection) collection() ColInfo {
|
||||
cat := path.FilesCategory
|
||||
if c.service == path.SharePointService {
|
||||
cat = path.LibrariesCategory
|
||||
}
|
||||
|
||||
return ColInfo{
|
||||
PathElements: c.PathElements,
|
||||
Category: cat,
|
||||
Items: c.items,
|
||||
AuxItems: c.aux,
|
||||
}
|
||||
}
|
||||
|
||||
func NewOneDriveCollection(
|
||||
service path.ServiceType,
|
||||
PathElements []string,
|
||||
backupVersion int,
|
||||
) *onedriveCollection {
|
||||
return &onedriveCollection{
|
||||
service: service,
|
||||
PathElements: PathElements,
|
||||
backupVersion: backupVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func DataForInfo(
|
||||
service path.ServiceType,
|
||||
cols []OnedriveColInfo,
|
||||
backupVersion int,
|
||||
) ([]ColInfo, error) {
|
||||
var (
|
||||
res []ColInfo
|
||||
err error
|
||||
)
|
||||
|
||||
for _, c := range cols {
|
||||
onedriveCol := NewOneDriveCollection(service, c.PathElements, backupVersion)
|
||||
|
||||
for _, f := range c.Files {
|
||||
_, err = onedriveCol.withFile(f.Name, f.Data, f.Perms)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, d := range c.Folders {
|
||||
_, err = onedriveCol.withFolder(d.Name, d.Perms)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = onedriveCol.withPermissions(c.Perms)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res = append(res, onedriveCol.collection())
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermData) (*onedriveCollection, error) {
|
||||
switch c.backupVersion {
|
||||
case 0:
|
||||
// Lookups will occur using the most recent version of things so we need
|
||||
// the embedded file name to match that.
|
||||
item, err := onedriveItemWithData(
|
||||
name,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
c.items = append(c.items, item)
|
||||
|
||||
// v1-5, early metadata design
|
||||
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker,
|
||||
version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName:
|
||||
items, err := onedriveItemWithData(
|
||||
name+metadata.DataFileSuffix,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
c.items = append(c.items, items)
|
||||
|
||||
md, err := onedriveMetadata(
|
||||
"",
|
||||
name+metadata.MetaFileSuffix,
|
||||
name+metadata.MetaFileSuffix,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
// v6+ current metadata design
|
||||
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
|
||||
item, err := onedriveItemWithData(
|
||||
name+metadata.DataFileSuffix,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
c.items = append(c.items, item)
|
||||
|
||||
md, err := onedriveMetadata(
|
||||
name,
|
||||
name+metadata.MetaFileSuffix,
|
||||
name,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
default:
|
||||
return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.backupVersion))
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCollection, error) {
|
||||
switch c.backupVersion {
|
||||
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
|
||||
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
|
||||
return c, nil
|
||||
|
||||
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
|
||||
item, err := onedriveMetadata(
|
||||
"",
|
||||
name+metadata.DirMetaFileSuffix,
|
||||
name+metadata.DirMetaFileSuffix,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
|
||||
c.items = append(c.items, item)
|
||||
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
default:
|
||||
return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.backupVersion))
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// withPermissions adds permissions to the folder represented by this
|
||||
// onedriveCollection.
|
||||
func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection, error) {
|
||||
// These versions didn't store permissions for the folder or didn't store them
|
||||
// in the folder's collection.
|
||||
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
name := c.PathElements[len(c.PathElements)-1]
|
||||
metaName := name
|
||||
|
||||
if c.backupVersion >= version.OneDrive5DirMetaNoName {
|
||||
// We switched to just .dirmeta for metadata file names.
|
||||
metaName = ""
|
||||
}
|
||||
|
||||
if name == odConsts.RootPathDir {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
md, err := onedriveMetadata(
|
||||
name,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
perm,
|
||||
c.backupVersion >= versionPermissionSwitchedToID)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
return c, err
|
||||
}
|
||||
|
||||
type testOneDriveData struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Data []byte `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func onedriveItemWithData(
|
||||
name, lookupKey string,
|
||||
fileData []byte,
|
||||
) (ItemInfo, error) {
|
||||
content := testOneDriveData{
|
||||
FileName: lookupKey,
|
||||
Data: fileData,
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(content)
|
||||
if err != nil {
|
||||
return ItemInfo{}, clues.Stack(err)
|
||||
}
|
||||
|
||||
return ItemInfo{
|
||||
name: name,
|
||||
data: serialized,
|
||||
lookupKey: lookupKey,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func onedriveMetadata(
|
||||
fileName, itemID, lookupKey string,
|
||||
perm PermData,
|
||||
permUseID bool,
|
||||
) (ItemInfo, error) {
|
||||
testMeta := getMetadata(fileName, perm, permUseID)
|
||||
|
||||
testMetaJSON, err := json.Marshal(testMeta)
|
||||
if err != nil {
|
||||
return ItemInfo{}, clues.Wrap(err, "marshalling metadata")
|
||||
}
|
||||
|
||||
return ItemInfo{
|
||||
name: itemID,
|
||||
data: testMetaJSON,
|
||||
lookupKey: lookupKey,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func GetCollectionsAndExpected(
|
||||
config ConfigInfo,
|
||||
testCollections []ColInfo,
|
||||
backupVersion int,
|
||||
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
|
||||
var (
|
||||
collections []data.RestoreCollection
|
||||
expectedData = map[string]map[string][]byte{}
|
||||
totalItems = 0
|
||||
totalKopiaItems = 0
|
||||
)
|
||||
|
||||
for _, owner := range config.ResourceOwners {
|
||||
numItems, kopiaItems, ownerCollections, userExpectedData, err := collectionsForInfo(
|
||||
config.Service,
|
||||
config.Tenant,
|
||||
owner,
|
||||
config.Dest,
|
||||
testCollections,
|
||||
backupVersion,
|
||||
)
|
||||
if err != nil {
|
||||
return totalItems, totalKopiaItems, collections, expectedData, err
|
||||
}
|
||||
|
||||
collections = append(collections, ownerCollections...)
|
||||
totalItems += numItems
|
||||
totalKopiaItems += kopiaItems
|
||||
|
||||
maps.Copy(expectedData, userExpectedData)
|
||||
}
|
||||
|
||||
return totalItems, totalKopiaItems, collections, expectedData, nil
|
||||
}
|
||||
@ -11,7 +11,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
|
||||
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
|
||||
@ -407,65 +406,30 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
|
||||
// Exchange Functions
|
||||
//-------------------------------------------------------------
|
||||
|
||||
func getCollectionsAndExpected(
|
||||
t *testing.T,
|
||||
config configInfo,
|
||||
testCollections []colInfo,
|
||||
backupVersion int,
|
||||
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
collections []data.RestoreCollection
|
||||
expectedData = map[string]map[string][]byte{}
|
||||
totalItems = 0
|
||||
totalKopiaItems = 0
|
||||
)
|
||||
|
||||
for _, owner := range config.resourceOwners {
|
||||
numItems, kopiaItems, ownerCollections, userExpectedData := collectionsForInfo(
|
||||
t,
|
||||
config.service,
|
||||
config.tenant,
|
||||
owner,
|
||||
config.dest,
|
||||
testCollections,
|
||||
backupVersion)
|
||||
|
||||
collections = append(collections, ownerCollections...)
|
||||
totalItems += numItems
|
||||
totalKopiaItems += kopiaItems
|
||||
|
||||
maps.Copy(expectedData, userExpectedData)
|
||||
}
|
||||
|
||||
return totalItems, totalKopiaItems, collections, expectedData
|
||||
}
|
||||
|
||||
func runRestore(
|
||||
t *testing.T,
|
||||
ctx context.Context, //revive:disable-line:context-as-argument
|
||||
config configInfo,
|
||||
config ConfigInfo,
|
||||
backupVersion int,
|
||||
collections []data.RestoreCollection,
|
||||
numRestoreItems int,
|
||||
) {
|
||||
t.Logf(
|
||||
"Restoring collections to %s for resourceOwners(s) %v\n",
|
||||
config.dest.ContainerName,
|
||||
config.resourceOwners)
|
||||
config.Dest.ContainerName,
|
||||
config.ResourceOwners)
|
||||
|
||||
start := time.Now()
|
||||
|
||||
restoreGC := loadConnector(ctx, t, config.resource)
|
||||
restoreSel := getSelectorWith(t, config.service, config.resourceOwners, true)
|
||||
restoreGC := loadConnector(ctx, t, config.Resource)
|
||||
restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true)
|
||||
deets, err := restoreGC.ConsumeRestoreCollections(
|
||||
ctx,
|
||||
backupVersion,
|
||||
config.acct,
|
||||
config.Acct,
|
||||
restoreSel,
|
||||
config.dest,
|
||||
config.opts,
|
||||
config.Dest,
|
||||
config.Opts,
|
||||
collections,
|
||||
fault.New(true))
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
@ -489,30 +453,30 @@ func runRestore(
|
||||
func runBackupAndCompare(
|
||||
t *testing.T,
|
||||
ctx context.Context, //revive:disable-line:context-as-argument
|
||||
config configInfo,
|
||||
config ConfigInfo,
|
||||
expectedData map[string]map[string][]byte,
|
||||
totalItems int,
|
||||
totalKopiaItems int,
|
||||
inputCollections []colInfo,
|
||||
inputCollections []ColInfo,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
// Run a backup and compare its output with what we put in.
|
||||
cats := make(map[path.CategoryType]struct{}, len(inputCollections))
|
||||
for _, c := range inputCollections {
|
||||
cats[c.category] = struct{}{}
|
||||
cats[c.Category] = struct{}{}
|
||||
}
|
||||
|
||||
var (
|
||||
expectedDests = make([]destAndCats, 0, len(config.resourceOwners))
|
||||
expectedDests = make([]destAndCats, 0, len(config.ResourceOwners))
|
||||
idToName = map[string]string{}
|
||||
nameToID = map[string]string{}
|
||||
)
|
||||
|
||||
for _, ro := range config.resourceOwners {
|
||||
for _, ro := range config.ResourceOwners {
|
||||
expectedDests = append(expectedDests, destAndCats{
|
||||
resourceOwner: ro,
|
||||
dest: config.dest.ContainerName,
|
||||
dest: config.Dest.ContainerName,
|
||||
cats: cats,
|
||||
})
|
||||
|
||||
@ -520,10 +484,10 @@ func runBackupAndCompare(
|
||||
nameToID[ro] = ro
|
||||
}
|
||||
|
||||
backupGC := loadConnector(ctx, t, config.resource)
|
||||
backupGC := loadConnector(ctx, t, config.Resource)
|
||||
backupGC.IDNameLookup = inMock.NewCache(idToName, nameToID)
|
||||
|
||||
backupSel := backupSelectorForExpected(t, config.service, expectedDests)
|
||||
backupSel := backupSelectorForExpected(t, config.Service, expectedDests)
|
||||
t.Logf("Selective backup of %s\n", backupSel)
|
||||
|
||||
start := time.Now()
|
||||
@ -533,7 +497,7 @@ func runBackupAndCompare(
|
||||
backupSel,
|
||||
nil,
|
||||
version.NoBackup,
|
||||
config.opts,
|
||||
config.Opts,
|
||||
fault.New(true))
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
// No excludes yet because this isn't an incremental backup.
|
||||
@ -570,22 +534,23 @@ func runRestoreBackupTest(
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
config := configInfo{
|
||||
acct: acct,
|
||||
opts: opts,
|
||||
resource: test.resource,
|
||||
service: test.service,
|
||||
tenant: tenant,
|
||||
resourceOwners: resourceOwners,
|
||||
dest: tester.DefaultTestRestoreDestination(""),
|
||||
config := ConfigInfo{
|
||||
Acct: acct,
|
||||
Opts: opts,
|
||||
Resource: test.resource,
|
||||
Service: test.service,
|
||||
Tenant: tenant,
|
||||
ResourceOwners: resourceOwners,
|
||||
Dest: tester.DefaultTestRestoreDestination(""),
|
||||
}
|
||||
|
||||
totalItems, totalKopiaItems, collections, expectedData := getCollectionsAndExpected(
|
||||
t,
|
||||
totalItems, totalKopiaItems, collections, expectedData, err := GetCollectionsAndExpected(
|
||||
config,
|
||||
test.collections,
|
||||
version.Backup)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
runRestore(
|
||||
t,
|
||||
ctx,
|
||||
@ -616,21 +581,21 @@ func runRestoreTestWithVerion(
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
config := configInfo{
|
||||
acct: acct,
|
||||
opts: opts,
|
||||
resource: test.resource,
|
||||
service: test.service,
|
||||
tenant: tenant,
|
||||
resourceOwners: resourceOwners,
|
||||
dest: tester.DefaultTestRestoreDestination(""),
|
||||
config := ConfigInfo{
|
||||
Acct: acct,
|
||||
Opts: opts,
|
||||
Resource: test.resource,
|
||||
Service: test.service,
|
||||
Tenant: tenant,
|
||||
ResourceOwners: resourceOwners,
|
||||
Dest: tester.DefaultTestRestoreDestination(""),
|
||||
}
|
||||
|
||||
totalItems, _, collections, _ := getCollectionsAndExpected(
|
||||
t,
|
||||
totalItems, _, collections, _, err := GetCollectionsAndExpected(
|
||||
config,
|
||||
test.collectionsPrevious,
|
||||
test.backupVersion)
|
||||
require.NoError(t, err)
|
||||
|
||||
runRestore(
|
||||
t,
|
||||
@ -655,21 +620,21 @@ func runRestoreBackupTestVersions(
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
config := configInfo{
|
||||
acct: acct,
|
||||
opts: opts,
|
||||
resource: test.resource,
|
||||
service: test.service,
|
||||
tenant: tenant,
|
||||
resourceOwners: resourceOwners,
|
||||
dest: tester.DefaultTestRestoreDestination(""),
|
||||
config := ConfigInfo{
|
||||
Acct: acct,
|
||||
Opts: opts,
|
||||
Resource: test.resource,
|
||||
Service: test.service,
|
||||
Tenant: tenant,
|
||||
ResourceOwners: resourceOwners,
|
||||
Dest: tester.DefaultTestRestoreDestination(""),
|
||||
}
|
||||
|
||||
totalItems, _, collections, _ := getCollectionsAndExpected(
|
||||
t,
|
||||
totalItems, _, collections, _, err := GetCollectionsAndExpected(
|
||||
config,
|
||||
test.collectionsPrevious,
|
||||
test.backupVersion)
|
||||
require.NoError(t, err)
|
||||
|
||||
runRestore(
|
||||
t,
|
||||
@ -680,11 +645,11 @@ func runRestoreBackupTestVersions(
|
||||
totalItems)
|
||||
|
||||
// Get expected output for new version.
|
||||
totalItems, totalKopiaItems, _, expectedData := getCollectionsAndExpected(
|
||||
t,
|
||||
totalItems, totalKopiaItems, _, expectedData, err := GetCollectionsAndExpected(
|
||||
config,
|
||||
test.collectionsLatest,
|
||||
version.Backup)
|
||||
require.NoError(t, err)
|
||||
|
||||
runBackupAndCompare(
|
||||
t,
|
||||
@ -705,11 +670,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
name: "EmailsWithAttachments",
|
||||
service: path.ExchangeService,
|
||||
resource: Users,
|
||||
collections: []colInfo{
|
||||
collections: []ColInfo{
|
||||
{
|
||||
pathElements: []string{"Inbox"},
|
||||
category: path.EmailCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Inbox"},
|
||||
Category: path.EmailCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID",
|
||||
data: exchMock.MessageWithDirectAttachment(
|
||||
@ -732,11 +697,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
name: "MultipleEmailsMultipleFolders",
|
||||
service: path.ExchangeService,
|
||||
resource: Users,
|
||||
collections: []colInfo{
|
||||
collections: []ColInfo{
|
||||
{
|
||||
pathElements: []string{"Inbox"},
|
||||
category: path.EmailCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Inbox"},
|
||||
Category: path.EmailCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID",
|
||||
data: exchMock.MessageWithBodyBytes(
|
||||
@ -749,9 +714,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: []string{"Work"},
|
||||
category: path.EmailCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Work"},
|
||||
Category: path.EmailCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID2",
|
||||
data: exchMock.MessageWithBodyBytes(
|
||||
@ -773,9 +738,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: []string{"Work", "Inbox"},
|
||||
category: path.EmailCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Work", "Inbox"},
|
||||
Category: path.EmailCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID4",
|
||||
data: exchMock.MessageWithBodyBytes(
|
||||
@ -788,9 +753,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: []string{"Work", "Inbox", "Work"},
|
||||
category: path.EmailCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Work", "Inbox", "Work"},
|
||||
Category: path.EmailCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID5",
|
||||
data: exchMock.MessageWithBodyBytes(
|
||||
@ -808,11 +773,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
name: "MultipleContactsSingleFolder",
|
||||
service: path.ExchangeService,
|
||||
resource: Users,
|
||||
collections: []colInfo{
|
||||
collections: []ColInfo{
|
||||
{
|
||||
pathElements: []string{"Contacts"},
|
||||
category: path.ContactsCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Contacts"},
|
||||
Category: path.ContactsCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID",
|
||||
data: exchMock.ContactBytes("Ghimley"),
|
||||
@ -836,11 +801,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
name: "MultipleContactsMultipleFolders",
|
||||
service: path.ExchangeService,
|
||||
resource: Users,
|
||||
collections: []colInfo{
|
||||
collections: []ColInfo{
|
||||
{
|
||||
pathElements: []string{"Work"},
|
||||
category: path.ContactsCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Work"},
|
||||
Category: path.ContactsCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID",
|
||||
data: exchMock.ContactBytes("Ghimley"),
|
||||
@ -859,9 +824,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: []string{"Personal"},
|
||||
category: path.ContactsCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Personal"},
|
||||
Category: path.ContactsCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID4",
|
||||
data: exchMock.ContactBytes("Argon"),
|
||||
@ -971,11 +936,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
||||
name: "Contacts",
|
||||
service: path.ExchangeService,
|
||||
resource: Users,
|
||||
collections: []colInfo{
|
||||
collections: []ColInfo{
|
||||
{
|
||||
pathElements: []string{"Work"},
|
||||
category: path.ContactsCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Work"},
|
||||
Category: path.ContactsCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID",
|
||||
data: exchMock.ContactBytes("Ghimley"),
|
||||
@ -984,9 +949,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: []string{"Personal"},
|
||||
category: path.ContactsCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Personal"},
|
||||
Category: path.ContactsCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "someencodeditemID2",
|
||||
data: exchMock.ContactBytes("Irgot"),
|
||||
@ -1012,9 +977,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
||||
// },
|
||||
// },
|
||||
// {
|
||||
// pathElements: []string{"Personal"},
|
||||
// category: path.EventsCategory,
|
||||
// items: []itemInfo{
|
||||
// PathElements: []string{"Personal"},
|
||||
// Category: path.EventsCategory,
|
||||
// Items: []ItemInfo{
|
||||
// {
|
||||
// name: "someencodeditemID2",
|
||||
// data: exchMock.EventWithSubjectBytes("Irgot"),
|
||||
@ -1045,19 +1010,20 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
||||
resourceOwner: suite.user,
|
||||
dest: dest.ContainerName,
|
||||
cats: map[path.CategoryType]struct{}{
|
||||
collection.category: {},
|
||||
collection.Category: {},
|
||||
},
|
||||
})
|
||||
|
||||
totalItems, _, collections, expectedData := collectionsForInfo(
|
||||
t,
|
||||
totalItems, _, collections, expectedData, err := collectionsForInfo(
|
||||
test.service,
|
||||
suite.connector.tenant,
|
||||
suite.user,
|
||||
dest,
|
||||
[]colInfo{collection},
|
||||
[]ColInfo{collection},
|
||||
version.Backup,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
allItems += totalItems
|
||||
|
||||
for k, v := range expectedData {
|
||||
@ -1123,10 +1089,10 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
||||
|
||||
t.Log("Backup enumeration complete")
|
||||
|
||||
ci := configInfo{
|
||||
opts: control.Options{RestorePermissions: true},
|
||||
ci := ConfigInfo{
|
||||
Opts: control.Options{RestorePermissions: true},
|
||||
// Alright to be empty, needed for OneDrive.
|
||||
dest: control.RestoreDestination{},
|
||||
Dest: control.RestoreDestination{},
|
||||
}
|
||||
|
||||
// Pull the data prior to waiting for the status as otherwise it will
|
||||
@ -1149,11 +1115,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup_largeMailAttac
|
||||
name: "EmailsWithLargeAttachments",
|
||||
service: path.ExchangeService,
|
||||
resource: Users,
|
||||
collections: []colInfo{
|
||||
collections: []ColInfo{
|
||||
{
|
||||
pathElements: []string{"Inbox"},
|
||||
category: path.EmailCategory,
|
||||
items: []itemInfo{
|
||||
PathElements: []string{"Inbox"},
|
||||
Category: path.EmailCategory,
|
||||
Items: []ItemInfo{
|
||||
{
|
||||
name: "35mbAttachment",
|
||||
data: exchMock.MessageWithSizedAttachment(subjectText, 35),
|
||||
|
||||
188
src/internal/connector/graph_connector_test_helper.go
Normal file
188
src/internal/connector/graph_connector_test_helper.go
Normal file
@ -0,0 +1,188 @@
|
||||
package connector
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
|
||||
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
type ColInfo struct {
|
||||
// Elements (in order) for the path representing this collection. Should
|
||||
// only contain elements after the prefix that corso uses for the path. For
|
||||
// example, a collection for the Inbox folder in exchange mail would just be
|
||||
// "Inbox".
|
||||
PathElements []string
|
||||
Category path.CategoryType
|
||||
Items []ItemInfo
|
||||
// auxItems are items that can be retrieved with Fetch but won't be returned
|
||||
// by Items(). These files do not directly participate in comparisosn at the
|
||||
// end of a test.
|
||||
AuxItems []ItemInfo
|
||||
}
|
||||
|
||||
type ItemInfo struct {
|
||||
// lookupKey is a string that can be used to find this data from a set of
|
||||
// other data in the same collection. This key should be something that will
|
||||
// be the same before and after restoring the item in M365 and may not be
|
||||
// the M365 ID. When restoring items out of place, the item is assigned a
|
||||
// new ID making it unsuitable for a lookup key.
|
||||
lookupKey string
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type ConfigInfo struct {
|
||||
Acct account.Account
|
||||
Opts control.Options
|
||||
Resource Resource
|
||||
Service path.ServiceType
|
||||
Tenant string
|
||||
ResourceOwners []string
|
||||
Dest control.RestoreDestination
|
||||
}
|
||||
|
||||
func mustToDataLayerPath(
|
||||
service path.ServiceType,
|
||||
tenant, resourceOwner string,
|
||||
category path.CategoryType,
|
||||
elements []string,
|
||||
isItem bool,
|
||||
) (path.Path, error) {
|
||||
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res, err
|
||||
}
|
||||
|
||||
// backupOutputPathFromRestore returns a path.Path denoting the location in
|
||||
// kopia the data will be placed at. The location is a data-type specific
|
||||
// combination of the location the data was recently restored to and where the
|
||||
// data was originally in the hierarchy.
|
||||
func backupOutputPathFromRestore(
|
||||
restoreDest control.RestoreDestination,
|
||||
inputPath path.Path,
|
||||
) (path.Path, error) {
|
||||
base := []string{restoreDest.ContainerName}
|
||||
|
||||
// OneDrive has leading information like the drive ID.
|
||||
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService {
|
||||
folders := inputPath.Folders()
|
||||
base = append(append([]string{}, folders[:3]...), restoreDest.ContainerName)
|
||||
|
||||
if len(folders) > 3 {
|
||||
base = append(base, folders[3:]...)
|
||||
}
|
||||
}
|
||||
|
||||
if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory {
|
||||
base = append(base, inputPath.Folders()...)
|
||||
}
|
||||
|
||||
return mustToDataLayerPath(
|
||||
inputPath.Service(),
|
||||
inputPath.Tenant(),
|
||||
inputPath.ResourceOwner(),
|
||||
inputPath.Category(),
|
||||
base,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
// TODO(ashmrtn): Make this an actual mock class that can be used in other
|
||||
// packages.
|
||||
type mockRestoreCollection struct {
|
||||
data.Collection
|
||||
auxItems map[string]data.Stream
|
||||
}
|
||||
|
||||
func (rc mockRestoreCollection) Fetch(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
) (data.Stream, error) {
|
||||
res := rc.auxItems[name]
|
||||
if res == nil {
|
||||
return nil, data.ErrNotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func collectionsForInfo(
|
||||
service path.ServiceType,
|
||||
tenant, user string,
|
||||
dest control.RestoreDestination,
|
||||
allInfo []ColInfo,
|
||||
backupVersion int,
|
||||
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
|
||||
var (
|
||||
collections = make([]data.RestoreCollection, 0, len(allInfo))
|
||||
expectedData = make(map[string]map[string][]byte, len(allInfo))
|
||||
totalItems = 0
|
||||
kopiaEntries = 0
|
||||
)
|
||||
|
||||
for _, info := range allInfo {
|
||||
pth, err := mustToDataLayerPath(
|
||||
service,
|
||||
tenant,
|
||||
user,
|
||||
info.Category,
|
||||
info.PathElements,
|
||||
false)
|
||||
if err != nil {
|
||||
return totalItems, kopiaEntries, collections, expectedData, err
|
||||
}
|
||||
|
||||
mc := exchMock.NewCollection(pth, pth, len(info.Items))
|
||||
|
||||
baseDestPath, err := backupOutputPathFromRestore(dest, pth)
|
||||
if err != nil {
|
||||
return totalItems, kopiaEntries, collections, expectedData, err
|
||||
}
|
||||
|
||||
baseExpected := expectedData[baseDestPath.String()]
|
||||
if baseExpected == nil {
|
||||
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items))
|
||||
baseExpected = expectedData[baseDestPath.String()]
|
||||
}
|
||||
|
||||
for i := 0; i < len(info.Items); i++ {
|
||||
mc.Names[i] = info.Items[i].name
|
||||
mc.Data[i] = info.Items[i].data
|
||||
|
||||
baseExpected[info.Items[i].lookupKey] = info.Items[i].data
|
||||
|
||||
// We do not count metadata files against item count
|
||||
if backupVersion > 0 &&
|
||||
(service == path.OneDriveService || service == path.SharePointService) &&
|
||||
metadata.HasMetaSuffix(info.Items[i].name) {
|
||||
continue
|
||||
}
|
||||
|
||||
totalItems++
|
||||
}
|
||||
|
||||
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
|
||||
|
||||
for _, aux := range info.AuxItems {
|
||||
c.auxItems[aux.name] = &exchMock.Data{
|
||||
ID: aux.name,
|
||||
Reader: io.NopCloser(bytes.NewReader(aux.data)),
|
||||
}
|
||||
}
|
||||
|
||||
collections = append(collections, c)
|
||||
kopiaEntries += len(info.Items)
|
||||
}
|
||||
|
||||
return totalItems, kopiaEntries, collections, expectedData, nil
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user