larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor onedrive test data creation

Browse Source
This commit is contained in:
neha-Gupta1 2023-05-08 15:32:28 +05:30
parent 1dcdd28a90
commit 42f23694fc
6 changed files with 1152 additions and 1105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

696
src/cmd/factory/impl/common.go
View File

@ -1,13 +1,11 @@
package impl package impl
import ( import (
"bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io"
"os" "os"
"strings" "strings"
"testing"
"time" "time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -20,7 +18,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector"
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
@ -82,7 +79,7 @@ func generateAndRestoreItems(
} }
collections := []collection{{ collections := []collection{{
pathElements: []string{destFldr}, PathElements: []string{destFldr},
category: cat, category: cat,
items: items, items: items,
}} }}
@ -161,7 +158,7 @@ type collection struct {
// only contain elements after the prefix that corso uses for the path. For // only contain elements after the prefix that corso uses for the path. For
// example, a collection for the Inbox folder in exchange mail would just be // example, a collection for the Inbox folder in exchange mail would just be
// "Inbox". // "Inbox".
pathElements []string PathElements []string
category path.CategoryType category path.CategoryType
items []item items []item
} }
@ -181,7 +178,7 @@ func buildCollections(
service, service,
c.category, c.category,
false, false,
c.pathElements...) c.PathElements...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -199,44 +196,44 @@ func buildCollections(
return collections, nil return collections, nil
} }
type permData struct { // type connector.PermData struct {
user string // user is only for older versions // user string // user is only for older versions
entityID string // entityID string
roles []string // roles []string
sharingMode metadata.SharingMode // sharingMode metadata.SharingMode
} // }
type itemData struct { // type connector.ItemData struct {
name string // name string
data []byte // data []byte
perms permData // perms connector.PermData
} // }
type itemInfo struct { // type itemInfo struct {
// lookupKey is a string that can be used to find this data from a set of // // lookupKey is a string that can be used to find this data from a set of
// other data in the same collection. This key should be something that will // // other data in the same collection. This key should be something that will
// be the same before and after restoring the item in M365 and may not be // // be the same before and after restoring the item in M365 and may not be
// the M365 ID. When restoring items out of place, the item is assigned a // // the M365 ID. When restoring items out of place, the item is assigned a
// new ID making it unsuitable for a lookup key. // // new ID making it unsuitable for a lookup key.
lookupKey string // lookupKey string
name string // name string
data []byte // data []byte
} // }
type onedriveCollection struct { // type onedriveCollection struct {
service path.ServiceType // service path.ServiceType
pathElements []string // PathElements []string
items []itemInfo // items []itemInfo
aux []itemInfo // aux []itemInfo
backupVersion int // backupVersion int
} // }
type onedriveColInfo struct { // type connector.OnedriveColInfo struct {
pathElements []string // PathElements []string
perms permData // perms connector.PermData
files []itemData // Files []connector.ItemData
folders []itemData // folders []connector.ItemData
} // }
var ( var (
folderAName = "folder-a" folderAName = "folder-a"
@ -277,7 +274,7 @@ func generateAndRestoreOnedriveItems(
driveID := ptr.Val(d.GetId()) driveID := ptr.Val(d.GetId())
var ( var (
cols []onedriveColInfo cols []connector.OnedriveColInfo
rootPath = []string{"drives", driveID, "root:"} rootPath = []string{"drives", driveID, "root:"}
folderAPath = []string{"drives", driveID, "root:", folderAName} folderAPath = []string{"drives", driveID, "root:", folderAName}
@ -291,43 +288,43 @@ func generateAndRestoreOnedriveItems(
) )
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
col := []onedriveColInfo{ col := []connector.OnedriveColInfo{
// basic folder and file creation // basic folder and file creation
{ {
pathElements: rootPath, PathElements: rootPath,
files: []itemData{ Files: []connector.ItemData{
{ {
name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
data: fileAData, Data: fileAData,
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
{ {
name: fmt.Sprintf("file-2nd-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-2nd-count-%d-at-%s", i, currentTime),
data: fileBData, Data: fileBData,
}, },
}, },
folders: []itemData{ Folders: []connector.ItemData{
{ {
name: folderBName, Name: folderBName,
}, },
{ {
name: folderAName, Name: folderAName,
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
{ {
name: folderCName, Name: folderCName,
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
}, },
@ -335,62 +332,62 @@ func generateAndRestoreOnedriveItems(
{ {
// a folder that has permissions with an item in the folder with // a folder that has permissions with an item in the folder with
// the different permissions. // the different permissions.
pathElements: folderAPath, PathElements: folderAPath,
files: []itemData{ Files: []connector.ItemData{
{ {
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
data: fileEData, Data: fileEData,
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
{ {
// a folder that has permissions with an item in the folder with // a folder that has permissions with an item in the folder with
// no permissions. // no permissions.
pathElements: folderCPath, PathElements: folderCPath,
files: []itemData{ Files: []connector.ItemData{
{ {
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
data: fileAData, Data: fileAData,
}, },
}, },
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
{ {
pathElements: folderBPath, PathElements: folderBPath,
files: []itemData{ Files: []connector.ItemData{
{ {
// restoring a file in a non-root folder that doesn't inherit // restoring a file in a non-root folder that doesn't inherit
// permissions. // permissions.
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
data: fileBData, Data: fileBData,
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
folders: []itemData{ Folders: []connector.ItemData{
{ {
name: folderAName, Name: folderAName,
perms: permData{ Perms: connector.PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
}, },
@ -400,302 +397,323 @@ func generateAndRestoreOnedriveItems(
cols = append(cols, col...) cols = append(cols, col...)
} }
input := dataForInfo(service, cols, version.Backup) // TODO Neha: work on this
t := testing.T{}
input := connector.DataForInfo(&t, service, cols, version.Backup)
collections := getCollections( // collections := getCollections(
service, // service,
tenantID, // tenantID,
[]string{resourceOwner}, // []string{resourceOwner},
input, // input,
version.Backup) // version.Backup)
opts := control.Options{ opts := control.Options{
RestorePermissions: true, RestorePermissions: true,
ToggleFeatures: control.Toggles{}, ToggleFeatures: control.Toggles{},
} }
config := connector.ConfigInfo{
Acct: acct,
Opts: opts,
Resource: connector.Users,
Service: service,
Tenant: tenantID,
ResourceOwners: []string{resourceOwner},
Dest: tester.DefaultTestRestoreDestination(),
}
_, _, collections, _ := connector.GetCollectionsAndExpected(
&t,
config,
// service,
// tenantID,
// []string{resourceOwner},
input,
version.Backup)
return gc.ConsumeRestoreCollections(ctx, version.Backup, acct, sel, dest, opts, collections, errs) return gc.ConsumeRestoreCollections(ctx, version.Backup, acct, sel, dest, opts, collections, errs)
} }
func getCollections( // func getCollections(
service path.ServiceType, // service path.ServiceType,
tenant string, // tenant string,
resourceOwners []string, // resourceOwners []string,
testCollections []colInfo, // testCollections []colInfo,
backupVersion int, // backupVersion int,
) []data.RestoreCollection { // ) []data.RestoreCollection {
var collections []data.RestoreCollection // var collections []data.RestoreCollection
for _, owner := range resourceOwners { // for _, owner := range resourceOwners {
ownerCollections := collectionsForInfo( // ownerCollections := collectionsForInfo(
service, // service,
tenant, // tenant,
owner, // owner,
testCollections, // testCollections,
backupVersion, // backupVersion,
) // )
collections = append(collections, ownerCollections...) // collections = append(collections, ownerCollections...)
} // }
return collections // return collections
} // }
type mockRestoreCollection struct { // type mockRestoreCollection struct {
data.Collection // data.Collection
auxItems map[string]data.Stream // auxItems map[string]data.Stream
} // }
func (rc mockRestoreCollection) Fetch( // func (rc mockRestoreCollection) Fetch(
ctx context.Context, // ctx context.Context,
name string, // name string,
) (data.Stream, error) { // ) (data.Stream, error) {
res := rc.auxItems[name] // res := rc.auxItems[name]
if res == nil { // if res == nil {
return nil, data.ErrNotFound // return nil, data.ErrNotFound
} // }
return res, nil // return res, nil
} // }
func collectionsForInfo( // func collectionsForInfo(
service path.ServiceType, // service path.ServiceType,
tenant, user string, // tenant, user string,
allInfo []colInfo, // allInfo []colInfo,
backupVersion int, // backupVersion int,
) []data.RestoreCollection { // ) []data.RestoreCollection {
collections := make([]data.RestoreCollection, 0, len(allInfo)) // collections := make([]data.RestoreCollection, 0, len(allInfo))
for _, info := range allInfo { // for _, info := range allInfo {
pth := mustToDataLayerPath( // pth := mustToDataLayerPath(
service, // service,
tenant, // tenant,
user, // user,
info.category, // info.category,
info.pathElements, // info.PathElements,
false) // false)
mc := exchMock.NewCollection(pth, pth, len(info.items)) // mc := exchMock.NewCollection(pth, pth, len(info.items))
for i := 0; i < len(info.items); i++ { // for i := 0; i < len(info.items); i++ {
mc.Names[i] = info.items[i].name // mc.Names[i] = info.items[i].name
mc.Data[i] = info.items[i].data // mc.Data[i] = info.items[i].data
// We do not count metadata files against item count // // We do not count metadata Files against item count
if backupVersion > 0 && metadata.HasMetaSuffix(info.items[i].name) && // if backupVersion > 0 && metadata.HasMetaSuffix(info.items[i].name) &&
(service == path.OneDriveService || service == path.SharePointService) { // (service == path.OneDriveService || service == path.SharePointService) {
continue // continue
} // }
} // }
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}} // c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.auxItems { // for _, aux := range info.auxItems {
c.auxItems[aux.name] = &exchMock.Data{ // c.auxItems[aux.name] = &exchMock.Data{
ID: aux.name, // ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)), // Reader: io.NopCloser(bytes.NewReader(aux.data)),
} // }
} // }
collections = append(collections, c) // collections = append(collections, c)
} // }
return collections // return collections
} // }
func mustToDataLayerPath( // func mustToDataLayerPath(
service path.ServiceType, // service path.ServiceType,
tenant, resourceOwner string, // tenant, resourceOwner string,
category path.CategoryType, // category path.CategoryType,
elements []string, // elements []string,
isItem bool, // isItem bool,
) path.Path { // ) path.Path {
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...) // res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
if err != nil { // if err != nil {
fmt.Println("building path", clues.ToCore(err)) // fmt.Println("building path", clues.ToCore(err))
} // }
return res // return res
} // }
type colInfo struct { // type colInfo struct {
// Elements (in order) for the path representing this collection. Should // // Elements (in order) for the path representing this collection. Should
// only contain elements after the prefix that corso uses for the path. For // // only contain elements after the prefix that corso uses for the path. For
// example, a collection for the Inbox folder in exchange mail would just be // // example, a collection for the Inbox folder in exchange mail would just be
// "Inbox". // // "Inbox".
pathElements []string // PathElements []string
category path.CategoryType // category path.CategoryType
items []itemInfo // items []itemInfo
// auxItems are items that can be retrieved with Fetch but won't be returned // // auxItems are items that can be retrieved with Fetch but won't be returned
// by Items(). // // by Items().
auxItems []itemInfo // auxItems []itemInfo
} // }
func newOneDriveCollection( // func newOneDriveCollection(
service path.ServiceType, // service path.ServiceType,
pathElements []string, // PathElements []string,
backupVersion int, // backupVersion int,
) *onedriveCollection { // ) *onedriveCollection {
return &onedriveCollection{ // return &onedriveCollection{
service: service, // service: service,
pathElements: pathElements, // PathElements: PathElements,
backupVersion: backupVersion, // backupVersion: backupVersion,
} // }
} // }
func dataForInfo( // func dataForInfo(
service path.ServiceType, // service path.ServiceType,
cols []onedriveColInfo, // cols []connector.OnedriveColInfo,
backupVersion int, // backupVersion int,
) []colInfo { // ) []colInfo {
var res []colInfo // var res []colInfo
for _, c := range cols { // for _, c := range cols {
onedriveCol := newOneDriveCollection(service, c.pathElements, backupVersion) // onedriveCol := newOneDriveCollection(service, c.PathElements, backupVersion)
for _, f := range c.files { // for _, f := range c.Files {
onedriveCol.withFile(f.name, f.data, f.perms) // onedriveCol.withFile(f.Name, f.Data, f.Perms)
} // }
onedriveCol.withPermissions(c.perms) // onedriveCol.withPermissions(c.Perms)
res = append(res, onedriveCol.collection()) // res = append(res, onedriveCol.collection())
} // }
return res // return res
} // }
func (c onedriveCollection) collection() colInfo { // func (c onedriveCollection) collection() colInfo {
cat := path.FilesCategory // cat := path.FilesCategory
if c.service == path.SharePointService { // if c.service == path.SharePointService {
cat = path.LibrariesCategory // cat = path.LibrariesCategory
} // }
return colInfo{ // return colInfo{
pathElements: c.pathElements, // PathElements: c.PathElements,
category: cat, // category: cat,
items: c.items, // items: c.items,
auxItems: c.aux, // auxItems: c.aux,
} // }
} // }
func (c *onedriveCollection) withFile(name string, fileData []byte, perm permData) *onedriveCollection { // func (c *onedriveCollection) withFile(name string, fileData []byte, perm connector.PermData) *onedriveCollection {
c.items = append(c.items, onedriveItemWithData( // c.items = append(c.items, onedriveItemWithData(
name+metadata.DataFileSuffix, // name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix, // name+metadata.DataFileSuffix,
fileData)) // fileData))
md := onedriveMetadata( // md := onedriveMetadata(
name, // name,
name+metadata.MetaFileSuffix, // name+metadata.MetaFileSuffix,
name, // name,
perm, // perm,
true) // true)
c.items = append(c.items, md) // c.items = append(c.items, md)
c.aux = append(c.aux, md) // c.aux = append(c.aux, md)
return c // return c
} // }
// withPermissions adds permissions to the folder represented by this // // withPermissions adds permissions to the folder represented by this
// onedriveCollection. // // onedriveCollection.
func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection { // func (c *onedriveCollection) withPermissions(perm connector.PermData) *onedriveCollection {
if c.backupVersion < version.OneDrive4DirIncludesPermissions { // if c.backupVersion < version.OneDrive4DirIncludesPermissions {
return c // return c
} // }
name := c.pathElements[len(c.pathElements)-1] // name := c.PathElements[len(c.PathElements)-1]
metaName := name // metaName := name
if c.backupVersion >= version.OneDrive5DirMetaNoName { // if c.backupVersion >= version.OneDrive5DirMetaNoName {
// We switched to just .dirmeta for metadata file names. // // We switched to just .dirmeta for metadata file names.
metaName = "" // metaName = ""
} // }
if name == "root:" { // if name == "root:" {
return c // return c
} // }
md := onedriveMetadata( // md := onedriveMetadata(
name, // name,
metaName+metadata.DirMetaFileSuffix, // metaName+metadata.DirMetaFileSuffix,
metaName+metadata.DirMetaFileSuffix, // metaName+metadata.DirMetaFileSuffix,
perm, // perm,
true) // true)
c.items = append(c.items, md) // c.items = append(c.items, md)
c.aux = append(c.aux, md) // c.aux = append(c.aux, md)
return c // return c
} // }
type oneDriveData struct { // type oneDriveData struct {
FileName string `json:"fileName,omitempty"` // FileName string `json:"fileName,omitempty"`
Data []byte `json:"data,omitempty"` // Data []byte `json:"data,omitempty"`
} // }
func onedriveItemWithData( // func onedriveItemWithData(
name, lookupKey string, // name, lookupKey string,
fileData []byte, // fileData []byte,
) itemInfo { // ) itemInfo {
content := oneDriveData{ // content := oneDriveData{
FileName: lookupKey, // FileName: lookupKey,
Data: fileData, // Data: fileData,
} // }
serialized, _ := json.Marshal(content) // serialized, _ := json.Marshal(content)
return itemInfo{ // return itemInfo{
name: name, // name: name,
data: serialized, // data: serialized,
lookupKey: lookupKey, // lookupKey: lookupKey,
} // }
} // }
func onedriveMetadata( // func onedriveMetadata(
fileName, itemID, lookupKey string, // fileName, itemID, lookupKey string,
perm permData, // perm connector.PermData,
permUseID bool, // permUseID bool,
) itemInfo { // ) itemInfo {
meta := getMetadata(fileName, perm, permUseID) // meta := getMetadata(fileName, perm, permUseID)
metaJSON, err := json.Marshal(meta) // metaJSON, err := json.Marshal(meta)
if err != nil { // if err != nil {
fmt.Println("marshalling metadata", clues.ToCore(err)) // fmt.Println("marshalling metadata", clues.ToCore(err))
} // }
return itemInfo{ // return itemInfo{
name: itemID, // name: itemID,
data: metaJSON, // data: metaJSON,
lookupKey: lookupKey, // lookupKey: lookupKey,
} // }
} // }
func getMetadata(fileName string, perm permData, permUseID bool) metadata.Metadata { // func getMetadata(fileName string, perm connector.PermData, permUseID bool) metadata.Metadata {
if len(perm.user) == 0 || len(perm.roles) == 0 || // if len(perm.User) == 0 || len(perm.Roles) == 0 ||
perm.sharingMode != metadata.SharingModeCustom { // perm.SharingMode != metadata.SharingModeCustom {
return metadata.Metadata{ // return metadata.Metadata{
FileName: fileName, // FileName: fileName,
SharingMode: perm.sharingMode, // SharingMode: perm.SharingMode,
} // }
} // }
// In case of permissions, the id will usually be same for same // // In case of permissions, the id will usually be same for same
// user/role combo unless deleted and readded, but we have to do // // user/role combo unless deleted and readded, but we have to do
// this as we only have two users of which one is already taken. // // this as we only have two users of which one is already taken.
id := uuid.NewString() // id := uuid.NewString()
uperm := metadata.Permission{ID: id, Roles: perm.roles} // uperm := metadata.Permission{ID: id, Roles: perm.Roles}
if permUseID { // if permUseID {
uperm.EntityID = perm.entityID // uperm.EntityID = perm.EntityID
} else { // } else {
uperm.Email = perm.user // uperm.Email = perm.User
} // }
meta := metadata.Metadata{ // meta := metadata.Metadata{
FileName: fileName, // FileName: fileName,
Permissions: []metadata.Permission{uperm}, // Permissions: []metadata.Permission{uperm},
} // }
return meta // return meta
} // }

183
src/internal/connector/graph_connector_helper_test.go
View File

@ -1,7 +1,6 @@
package connector package connector
import ( import (
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"io" "io"
@ -17,33 +16,17 @@ import (
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
"github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
) )
func mustToDataLayerPath(
t *testing.T,
service path.ServiceType,
tenant, resourceOwner string,
category path.CategoryType,
elements []string,
isItem bool,
) path.Path {
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
require.NoError(t, err, clues.ToCore(err))
return res
}
func testElementsMatch[T any]( func testElementsMatch[T any](
t *testing.T, t *testing.T,
expected []T, expected []T,
@ -100,7 +83,7 @@ func testElementsMatch[T any](
assert.Failf( assert.Failf(
t, t,
"contain different elements", "contain different elements",
"missing items: (%T)%v\nunexpected items: (%T)%v\n", "missing Items: (%T)%v\nunexpected Items: (%T)%v\n",
expected, expected,
missing, missing,
got, got,
@ -108,52 +91,17 @@ func testElementsMatch[T any](
) )
} }
type configInfo struct {
acct account.Account
opts control.Options
resource Resource
service path.ServiceType
tenant string
resourceOwners []string
dest control.RestoreDestination
}
type itemInfo struct {
// lookupKey is a string that can be used to find this data from a set of
// other data in the same collection. This key should be something that will
// be the same before and after restoring the item in M365 and may not be
// the M365 ID. When restoring items out of place, the item is assigned a
// new ID making it unsuitable for a lookup key.
lookupKey string
name string
data []byte
}
type colInfo struct {
// Elements (in order) for the path representing this collection. Should
// only contain elements after the prefix that corso uses for the path. For
// example, a collection for the Inbox folder in exchange mail would just be
// "Inbox".
pathElements []string
category path.CategoryType
items []itemInfo
// auxItems are items that can be retrieved with Fetch but won't be returned
// by Items(). These files do not directly participate in comparisosn at the
// end of a test.
auxItems []itemInfo
}
type restoreBackupInfo struct { type restoreBackupInfo struct {
name string name string
service path.ServiceType service path.ServiceType
collections []colInfo collections []ColInfo
resource Resource resource Resource
} }
type restoreBackupInfoMultiVersion struct { type restoreBackupInfoMultiVersion struct {
service path.ServiceType service path.ServiceType
collectionsLatest []colInfo collectionsLatest []ColInfo
collectionsPrevious []colInfo collectionsPrevious []ColInfo
resource Resource resource Resource
backupVersion int backupVersion int
} }
@ -1099,7 +1047,7 @@ func makeSharePointBackupSel(
} }
// backupSelectorForExpected creates a selector that can be used to backup the // backupSelectorForExpected creates a selector that can be used to backup the
// given items in expected based on the item paths. Fails the test if items from // given Items in expected based on the item paths. Fails the test if items from
// multiple services are in expected. // multiple services are in expected.
func backupSelectorForExpected( func backupSelectorForExpected(
t *testing.T, t *testing.T,
@ -1126,127 +1074,6 @@ func backupSelectorForExpected(
return selectors.Selector{} return selectors.Selector{}
} }
// backupOutputPathFromRestore returns a path.Path denoting the location in
// kopia the data will be placed at. The location is a data-type specific
// combination of the location the data was recently restored to and where the
// data was originally in the hierarchy.
func backupOutputPathFromRestore(
t *testing.T,
restoreDest control.RestoreDestination,
inputPath path.Path,
) path.Path {
base := []string{restoreDest.ContainerName}
// OneDrive has leading information like the drive ID.
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService {
folders := inputPath.Folders()
base = append(append([]string{}, folders[:3]...), restoreDest.ContainerName)
if len(folders) > 3 {
base = append(base, folders[3:]...)
}
}
if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory {
base = append(base, inputPath.Folders()...)
}
return mustToDataLayerPath(
t,
inputPath.Service(),
inputPath.Tenant(),
inputPath.ResourceOwner(),
inputPath.Category(),
base,
false,
)
}
// TODO(ashmrtn): Make this an actual mock class that can be used in other
// packages.
type mockRestoreCollection struct {
data.Collection
auxItems map[string]data.Stream
}
func (rc mockRestoreCollection) Fetch(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.auxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}
func collectionsForInfo(
t *testing.T,
service path.ServiceType,
tenant, user string,
dest control.RestoreDestination,
allInfo []colInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
var (
collections = make([]data.RestoreCollection, 0, len(allInfo))
expectedData = make(map[string]map[string][]byte, len(allInfo))
totalItems = 0
kopiaEntries = 0
)
for _, info := range allInfo {
pth := mustToDataLayerPath(
t,
service,
tenant,
user,
info.category,
info.pathElements,
false)
mc := exchMock.NewCollection(pth, pth, len(info.items))
baseDestPath := backupOutputPathFromRestore(t, dest, pth)
baseExpected := expectedData[baseDestPath.String()]
if baseExpected == nil {
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.items))
baseExpected = expectedData[baseDestPath.String()]
}
for i := 0; i < len(info.items); i++ {
mc.Names[i] = info.items[i].name
mc.Data[i] = info.items[i].data
baseExpected[info.items[i].lookupKey] = info.items[i].data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.items[i].name) {
continue
}
totalItems++
}
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.auxItems {
c.auxItems[aux.name] = &exchMock.Data{
ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.items)
}
return totalItems, kopiaEntries, collections, expectedData
}
func getSelectorWith( func getSelectorWith(
t *testing.T, t *testing.T,
service path.ServiceType, service path.ServiceType,

640
src/internal/connector/graph_connector_onedrive_test.go
View File

@ -2,13 +2,11 @@ package connector
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"strings" "strings"
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -24,92 +22,12 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
// For any version post this(inclusive), we expect to be using IDs for
// permission instead of email
const versionPermissionSwitchedToID = version.OneDrive4DirIncludesPermissions
func getMetadata(fileName string, perm permData, permUseID bool) metadata.Metadata {
if len(perm.user) == 0 || len(perm.roles) == 0 ||
perm.sharingMode != metadata.SharingModeCustom {
return metadata.Metadata{
FileName: fileName,
SharingMode: perm.sharingMode,
}
}
// In case of permissions, the id will usually be same for same
// user/role combo unless deleted and readded, but we have to do
// this as we only have two users of which one is already taken.
id := uuid.NewString()
uperm := metadata.Permission{ID: id, Roles: perm.roles}
if permUseID {
uperm.EntityID = perm.entityID
} else {
uperm.Email = perm.user
}
testMeta := metadata.Metadata{
FileName: fileName,
Permissions: []metadata.Permission{uperm},
}
return testMeta
}
type testOneDriveData struct {
FileName string `json:"fileName,omitempty"`
Data []byte `json:"data,omitempty"`
}
func onedriveItemWithData(
t *testing.T,
name, lookupKey string,
fileData []byte,
) itemInfo {
t.Helper()
content := testOneDriveData{
FileName: lookupKey,
Data: fileData,
}
serialized, err := json.Marshal(content)
require.NoError(t, err, clues.ToCore(err))
return itemInfo{
name: name,
data: serialized,
lookupKey: lookupKey,
}
}
func onedriveMetadata(
t *testing.T,
fileName, itemID, lookupKey string,
perm permData,
permUseID bool,
) itemInfo {
t.Helper()
testMeta := getMetadata(fileName, perm, permUseID)
testMetaJSON, err := json.Marshal(testMeta)
require.NoError(t, err, "marshalling metadata", clues.ToCore(err))
return itemInfo{
name: itemID,
data: testMetaJSON,
lookupKey: lookupKey,
}
}
var ( var (
fileName = "test-file.txt" fileName = "test-file.txt"
folderAName = "folder-a" folderAName = "folder-a"
folderBName = "b" folderBName = "b"
folderNamedFolder = "folder" folderNamedFolder = "folder"
rootFolder = "root:" // rootFolder = "root:"
fileAData = []byte(strings.Repeat("a", 33)) fileAData = []byte(strings.Repeat("a", 33))
fileBData = []byte(strings.Repeat("b", 65)) fileBData = []byte(strings.Repeat("b", 65))
@ -122,204 +40,6 @@ var (
readPerm = []string{"read"} readPerm = []string{"read"}
) )
func newOneDriveCollection(
t *testing.T,
service path.ServiceType,
pathElements []string,
backupVersion int,
) *onedriveCollection {
return &onedriveCollection{
service: service,
pathElements: pathElements,
backupVersion: backupVersion,
t: t,
}
}
type onedriveCollection struct {
service path.ServiceType
pathElements []string
items []itemInfo
aux []itemInfo
backupVersion int
t *testing.T
}
func (c onedriveCollection) collection() colInfo {
cat := path.FilesCategory
if c.service == path.SharePointService {
cat = path.LibrariesCategory
}
return colInfo{
pathElements: c.pathElements,
category: cat,
items: c.items,
auxItems: c.aux,
}
}
func (c *onedriveCollection) withFile(name string, fileData []byte, perm permData) *onedriveCollection {
switch c.backupVersion {
case 0:
// Lookups will occur using the most recent version of things so we need
// the embedded file name to match that.
c.items = append(c.items, onedriveItemWithData(
c.t,
name,
name+metadata.DataFileSuffix,
fileData))
// v1-5, early metadata design
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker,
version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName:
c.items = append(c.items, onedriveItemWithData(
c.t,
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData))
md := onedriveMetadata(
c.t,
"",
name+metadata.MetaFileSuffix,
name+metadata.MetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
// v6+ current metadata design
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
c.items = append(c.items, onedriveItemWithData(
c.t,
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData))
md := onedriveMetadata(
c.t,
name,
name+metadata.MetaFileSuffix,
name,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
default:
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
}
return c
}
func (c *onedriveCollection) withFolder(name string, perm permData) *onedriveCollection {
switch c.backupVersion {
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
return c
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
c.items = append(
c.items,
onedriveMetadata(
c.t,
"",
name+metadata.DirMetaFileSuffix,
name+metadata.DirMetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID))
default:
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
}
return c
}
// withPermissions adds permissions to the folder represented by this
// onedriveCollection.
func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection {
// These versions didn't store permissions for the folder or didn't store them
// in the folder's collection.
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
return c
}
name := c.pathElements[len(c.pathElements)-1]
metaName := name
if c.backupVersion >= version.OneDrive5DirMetaNoName {
// We switched to just .dirmeta for metadata file names.
metaName = ""
}
if name == rootFolder {
return c
}
md := onedriveMetadata(
c.t,
name,
metaName+metadata.DirMetaFileSuffix,
metaName+metadata.DirMetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
return c
}
type permData struct {
user string // user is only for older versions
entityID string
roles []string
sharingMode metadata.SharingMode
}
type itemData struct {
name string
data []byte
perms permData
}
type onedriveColInfo struct {
pathElements []string
perms permData
files []itemData
folders []itemData
}
func testDataForInfo(
t *testing.T,
service path.ServiceType,
cols []onedriveColInfo,
backupVersion int,
) []colInfo {
var res []colInfo
for _, c := range cols {
onedriveCol := newOneDriveCollection(t, service, c.pathElements, backupVersion)
for _, f := range c.files {
onedriveCol.withFile(f.name, f.data, f.perms)
}
for _, d := range c.folders {
onedriveCol.withFolder(d.name, d.perms)
}
onedriveCol.withPermissions(c.perms)
res = append(res, onedriveCol.collection())
}
return res
}
func mustGetDefaultDriveID( func mustGetDefaultDriveID(
t *testing.T, t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
@ -663,78 +383,78 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
folderBName, folderBName,
} }
cols := []onedriveColInfo{ cols := []OnedriveColInfo{
{ {
pathElements: rootPath, PathElements: rootPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileAData, Data: fileAData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderAName, Name: folderAName,
}, },
{ {
name: folderBName, Name: folderBName,
}, },
}, },
}, },
{ {
pathElements: folderAPath, PathElements: folderAPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileBData, Data: fileBData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderBName, Name: folderBName,
}, },
}, },
}, },
{ {
pathElements: subfolderBPath, PathElements: subfolderBPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileCData, Data: fileCData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderAName, Name: folderAName,
}, },
}, },
}, },
{ {
pathElements: subfolderAPath, PathElements: subfolderAPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileDData, Data: fileDData,
}, },
}, },
}, },
{ {
pathElements: folderBPath, PathElements: folderBPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileEData, Data: fileEData,
}, },
}, },
}, },
} }
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup) expected := DataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() { suite.Run(fmt.Sprintf("Version%d", vn), func() {
t := suite.T() t := suite.T()
input := testDataForInfo(t, suite.BackupService(), cols, vn) input := DataForInfo(t, suite.BackupService(), cols, vn)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.BackupService(), service: suite.BackupService(),
@ -807,71 +527,71 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
folderCName, folderCName,
} }
cols := []onedriveColInfo{ cols := []OnedriveColInfo{
{ {
pathElements: rootPath, PathElements: rootPath,
files: []itemData{ Files: []ItemData{
{ {
// Test restoring a file that doesn't inherit permissions. // Test restoring a file that doesn't inherit permissions.
name: fileName, Name: fileName,
data: fileAData, Data: fileAData,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
{ {
// Test restoring a file that doesn't inherit permissions and has // Test restoring a file that doesn't inherit permissions and has
// no permissions. // no permissions.
name: fileName2, Name: fileName2,
data: fileBData, Data: fileBData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderBName, Name: folderBName,
}, },
{ {
name: folderAName, Name: folderAName,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
{ {
name: folderCName, Name: folderCName,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
}, },
}, },
{ {
pathElements: folderBPath, PathElements: folderBPath,
files: []itemData{ Files: []ItemData{
{ {
// Test restoring a file in a non-root folder that doesn't inherit // Test restoring a file in a non-root folder that doesn't inherit
// permissions. // permissions.
name: fileName, Name: fileName,
data: fileBData, Data: fileBData,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderAName, Name: folderAName,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
}, },
@ -893,52 +613,52 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// }, // },
// }, // },
// }, // },
// perms: permData{ // Perms: PermData{
// user: secondaryUserName, // User: secondaryUserName,
// entityID: secondaryUserID, // EntityID: secondaryUserID,
// roles: readPerm, // Roles: readPerm,
// }, // },
// }, // },
{ {
// Tests a folder that has permissions with an item in the folder with // Tests a folder that has permissions with an item in the folder with
// the different permissions. // the different permissions.
pathElements: folderAPath, PathElements: folderAPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileEData, Data: fileEData,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
{ {
// Tests a folder that has permissions with an item in the folder with // Tests a folder that has permissions with an item in the folder with
// no permissions. // no permissions.
pathElements: folderCPath, PathElements: folderCPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileAData, Data: fileAData,
}, },
}, },
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
} }
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup) expected := DataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() { suite.Run(fmt.Sprintf("Version%d", vn), func() {
@ -946,7 +666,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Ideally this can always be true or false and still // Ideally this can always be true or false and still
// work, but limiting older versions to use emails so as // work, but limiting older versions to use emails so as
// to validate that flow as well. // to validate that flow as well.
input := testDataForInfo(t, suite.BackupService(), cols, vn) input := DataForInfo(t, suite.BackupService(), cols, vn)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.BackupService(), service: suite.BackupService(),
@ -984,50 +704,50 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
suite.Service(), suite.Service(),
suite.BackupResourceOwner()) suite.BackupResourceOwner())
inputCols := []onedriveColInfo{ inputCols := []OnedriveColInfo{
{ {
pathElements: []string{ PathElements: []string{
"drives", "drives",
driveID, driveID,
rootFolder, rootFolder,
}, },
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileAData, Data: fileAData,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
}, },
} }
expectedCols := []onedriveColInfo{ expectedCols := []OnedriveColInfo{
{ {
pathElements: []string{ PathElements: []string{
"drives", "drives",
driveID, driveID,
rootFolder, rootFolder,
}, },
files: []itemData{ Files: []ItemData{
{ {
// No permissions on the output since they weren't restored. // No permissions on the output since they weren't restored.
name: fileName, Name: fileName,
data: fileAData, Data: fileAData,
}, },
}, },
}, },
} }
expected := testDataForInfo(suite.T(), suite.BackupService(), expectedCols, version.Backup) expected := DataForInfo(suite.T(), suite.BackupService(), expectedCols, version.Backup)
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() { suite.Run(fmt.Sprintf("Version%d", vn), func() {
t := suite.T() t := suite.T()
input := testDataForInfo(t, suite.BackupService(), inputCols, vn) input := DataForInfo(t, suite.BackupService(), inputCols, vn)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.BackupService(), service: suite.BackupService(),
@ -1105,29 +825,29 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
folderCName, folderCName,
} }
fileSet := []itemData{ fileSet := []ItemData{
{ {
name: "file-custom", Name: "file-custom",
data: fileAData, Data: fileAData,
perms: permData{ Perms: PermData{
user: secondaryUserName, User: secondaryUserName,
entityID: secondaryUserID, EntityID: secondaryUserID,
roles: writePerm, Roles: writePerm,
sharingMode: metadata.SharingModeCustom, SharingMode: metadata.SharingModeCustom,
}, },
}, },
{ {
name: "file-inherited", Name: "file-inherited",
data: fileAData, Data: fileAData,
perms: permData{ Perms: PermData{
sharingMode: metadata.SharingModeInherited, SharingMode: metadata.SharingModeInherited,
}, },
}, },
{ {
name: "file-empty", Name: "file-empty",
data: fileAData, Data: fileAData,
perms: permData{ Perms: PermData{
sharingMode: metadata.SharingModeCustom, SharingMode: metadata.SharingModeCustom,
}, },
}, },
} }
@ -1150,55 +870,55 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
// - inherted-permission-file // - inherted-permission-file
// - empty-permission-file (empty/empty might have interesting behavior) // - empty-permission-file (empty/empty might have interesting behavior)
cols := []onedriveColInfo{ cols := []OnedriveColInfo{
{ {
pathElements: rootPath, PathElements: rootPath,
files: []itemData{}, Files: []ItemData{},
folders: []itemData{ Folders: []ItemData{
{name: folderAName}, {Name: folderAName},
}, },
}, },
{ {
pathElements: folderAPath, PathElements: folderAPath,
files: fileSet, Files: fileSet,
folders: []itemData{ Folders: []ItemData{
{name: folderAName}, {Name: folderAName},
{name: folderBName}, {Name: folderBName},
{name: folderCName}, {Name: folderCName},
}, },
perms: permData{ Perms: PermData{
user: tertiaryUserName, User: tertiaryUserName,
entityID: tertiaryUserID, EntityID: tertiaryUserID,
roles: readPerm, Roles: readPerm,
}, },
}, },
{ {
pathElements: subfolderAAPath, PathElements: subfolderAAPath,
files: fileSet, Files: fileSet,
perms: permData{ Perms: PermData{
user: tertiaryUserName, User: tertiaryUserName,
entityID: tertiaryUserID, EntityID: tertiaryUserID,
roles: writePerm, Roles: writePerm,
sharingMode: metadata.SharingModeCustom, SharingMode: metadata.SharingModeCustom,
}, },
}, },
{ {
pathElements: subfolderABPath, PathElements: subfolderABPath,
files: fileSet, Files: fileSet,
perms: permData{ Perms: PermData{
sharingMode: metadata.SharingModeInherited, SharingMode: metadata.SharingModeInherited,
}, },
}, },
{ {
pathElements: subfolderACPath, PathElements: subfolderACPath,
files: fileSet, Files: fileSet,
perms: permData{ Perms: PermData{
sharingMode: metadata.SharingModeCustom, SharingMode: metadata.SharingModeCustom,
}, },
}, },
} }
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup) expected := DataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() { suite.Run(fmt.Sprintf("Version%d", vn), func() {
@ -1206,7 +926,7 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
// Ideally this can always be true or false and still // Ideally this can always be true or false and still
// work, but limiting older versions to use emails so as // work, but limiting older versions to use emails so as
// to validate that flow as well. // to validate that flow as well.
input := testDataForInfo(t, suite.BackupService(), cols, vn) input := DataForInfo(t, suite.BackupService(), cols, vn)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.BackupService(), service: suite.BackupService(),
@ -1264,60 +984,60 @@ func testRestoreFolderNamedFolderRegression(
folderBName, folderBName,
} }
cols := []onedriveColInfo{ cols := []OnedriveColInfo{
{ {
pathElements: rootPath, PathElements: rootPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileAData, Data: fileAData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderNamedFolder, Name: folderNamedFolder,
}, },
{ {
name: folderBName, Name: folderBName,
}, },
}, },
}, },
{ {
pathElements: folderFolderPath, PathElements: folderFolderPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileBData, Data: fileBData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderBName, Name: folderBName,
}, },
}, },
}, },
{ {
pathElements: subfolderPath, PathElements: subfolderPath,
files: []itemData{ Files: []ItemData{
{ {
name: fileName, Name: fileName,
data: fileCData, Data: fileCData,
}, },
}, },
folders: []itemData{ Folders: []ItemData{
{ {
name: folderNamedFolder, Name: folderNamedFolder,
}, },
}, },
}, },
} }
expected := testDataForInfo(suite.T(), suite.BackupService(), cols, version.Backup) expected := DataForInfo(suite.T(), suite.BackupService(), cols, version.Backup)
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() { suite.Run(fmt.Sprintf("Version%d", vn), func() {
t := suite.T() t := suite.T()
input := testDataForInfo(t, suite.BackupService(), cols, vn) input := DataForInfo(t, suite.BackupService(), cols, vn)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.BackupService(), service: suite.BackupService(),

332
src/internal/connector/graph_connector_onedrive_test_helper.go Normal file
View File

@ -0,0 +1,332 @@
package connector
import (
"encoding/json"
"testing"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/maps"
)
// For any version post this(inclusive), we expect to be using IDs for
// permission instead of email
const versionPermissionSwitchedToID = version.OneDrive4DirIncludesPermissions
var rootFolder = "root:"
func getMetadata(fileName string, perm PermData, permUseID bool) metadata.Metadata {
if len(perm.User) == 0 || len(perm.Roles) == 0 ||
perm.SharingMode != metadata.SharingModeCustom {
return metadata.Metadata{
FileName: fileName,
SharingMode: perm.SharingMode,
}
}
// In case of permissions, the id will usually be same for same
// user/role combo unless deleted and readded, but we have to do
// this as we only have two users of which one is already taken.
id := uuid.NewString()
uperm := metadata.Permission{ID: id, Roles: perm.Roles}
if permUseID {
uperm.EntityID = perm.EntityID
} else {
uperm.Email = perm.User
}
testMeta := metadata.Metadata{
FileName: fileName,
Permissions: []metadata.Permission{uperm},
}
return testMeta
}
type PermData struct {
User string // user is only for older versions
EntityID string
Roles []string
SharingMode metadata.SharingMode
}
type ItemData struct {
Name string
Data []byte
Perms PermData
}
type OnedriveColInfo struct {
PathElements []string
Perms PermData
Files []ItemData
Folders []ItemData
}
type onedriveCollection struct {
service path.ServiceType
PathElements []string
items []ItemInfo
aux []ItemInfo
backupVersion int
t *testing.T
}
func (c onedriveCollection) collection() ColInfo {
cat := path.FilesCategory
if c.service == path.SharePointService {
cat = path.LibrariesCategory
}
return ColInfo{
PathElements: c.PathElements,
Category: cat,
Items: c.items,
AuxItems: c.aux,
}
}
func NewOneDriveCollection(
t *testing.T,
service path.ServiceType,
PathElements []string,
backupVersion int,
) *onedriveCollection {
return &onedriveCollection{
service: service,
PathElements: PathElements,
backupVersion: backupVersion,
t: t,
}
}
func DataForInfo(
t *testing.T,
service path.ServiceType,
cols []OnedriveColInfo,
backupVersion int,
) []ColInfo {
var res []ColInfo
for _, c := range cols {
onedriveCol := NewOneDriveCollection(t, service, c.PathElements, backupVersion)
for _, f := range c.Files {
onedriveCol.withFile(f.Name, f.Data, f.Perms)
}
for _, d := range c.Folders {
onedriveCol.withFolder(d.Name, d.Perms)
}
onedriveCol.withPermissions(c.Perms)
res = append(res, onedriveCol.collection())
}
return res
}
func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermData) *onedriveCollection {
switch c.backupVersion {
case 0:
// Lookups will occur using the most recent version of things so we need
// the embedded file name to match that.
c.items = append(c.items, onedriveItemWithData(
c.t,
name,
name+metadata.DataFileSuffix,
fileData))
// v1-5, early metadata design
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker,
version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName:
c.items = append(c.items, onedriveItemWithData(
c.t,
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData))
md := onedriveMetadata(
c.t,
"",
name+metadata.MetaFileSuffix,
name+metadata.MetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
// v6+ current metadata design
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
c.items = append(c.items, onedriveItemWithData(
c.t,
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData))
md := onedriveMetadata(
c.t,
name,
name+metadata.MetaFileSuffix,
name,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
default:
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
}
return c
}
func (c *onedriveCollection) withFolder(name string, perm PermData) *onedriveCollection {
switch c.backupVersion {
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
return c
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
c.items = append(
c.items,
onedriveMetadata(
c.t,
"",
name+metadata.DirMetaFileSuffix,
name+metadata.DirMetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID))
default:
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
}
return c
}
// withPermissions adds permissions to the folder represented by this
// onedriveCollection.
func (c *onedriveCollection) withPermissions(perm PermData) *onedriveCollection {
// These versions didn't store permissions for the folder or didn't store them
// in the folder's collection.
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
return c
}
name := c.PathElements[len(c.PathElements)-1]
metaName := name
if c.backupVersion >= version.OneDrive5DirMetaNoName {
// We switched to just .dirmeta for metadata file names.
metaName = ""
}
if name == rootFolder {
return c
}
md := onedriveMetadata(
c.t,
name,
metaName+metadata.DirMetaFileSuffix,
metaName+metadata.DirMetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
return c
}
type testOneDriveData struct {
FileName string `json:"fileName,omitempty"`
Data []byte `json:"data,omitempty"`
}
func onedriveItemWithData(
t *testing.T,
name, lookupKey string,
fileData []byte,
) ItemInfo {
t.Helper()
content := testOneDriveData{
FileName: lookupKey,
Data: fileData,
}
serialized, err := json.Marshal(content)
require.NoError(t, err, clues.ToCore(err))
return ItemInfo{
name: name,
data: serialized,
lookupKey: lookupKey,
}
}
func onedriveMetadata(
t *testing.T,
fileName, itemID, lookupKey string,
perm PermData,
permUseID bool,
) ItemInfo {
t.Helper()
testMeta := getMetadata(fileName, perm, permUseID)
testMetaJSON, err := json.Marshal(testMeta)
require.NoError(t, err, "marshalling metadata", clues.ToCore(err))
return ItemInfo{
name: itemID,
data: testMetaJSON,
lookupKey: lookupKey,
}
}
func GetCollectionsAndExpected(
t *testing.T,
config ConfigInfo,
testCollections []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
t.Helper()
var (
collections []data.RestoreCollection
expectedData = map[string]map[string][]byte{}
totalItems = 0
totalKopiaItems = 0
)
for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData := collectionsForInfo(
t,
config.Service,
config.Tenant,
owner,
config.Dest,
testCollections,
backupVersion,
)
collections = append(collections, ownerCollections...)
totalItems += numItems
totalKopiaItems += kopiaItems
maps.Copy(expectedData, userExpectedData)
}
return totalItems, totalKopiaItems, collections, expectedData
}

219
src/internal/connector/graph_connector_test.go
View File

@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"golang.org/x/exp/maps"
inMock "github.com/alcionai/corso/src/internal/common/idname/mock" inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
@ -407,66 +406,30 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
// Exchange Functions // Exchange Functions
//------------------------------------------------------------- //-------------------------------------------------------------
func getCollectionsAndExpected(
t *testing.T,
config configInfo,
testCollections []colInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
t.Helper()
var (
collections []data.RestoreCollection
expectedData = map[string]map[string][]byte{}
totalItems = 0
totalKopiaItems = 0
)
for _, owner := range config.resourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData := collectionsForInfo(
t,
config.service,
config.tenant,
owner,
config.dest,
testCollections,
backupVersion,
)
collections = append(collections, ownerCollections...)
totalItems += numItems
totalKopiaItems += kopiaItems
maps.Copy(expectedData, userExpectedData)
}
return totalItems, totalKopiaItems, collections, expectedData
}
func runRestore( func runRestore(
t *testing.T, t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
config configInfo, config ConfigInfo,
backupVersion int, backupVersion int,
collections []data.RestoreCollection, collections []data.RestoreCollection,
numRestoreItems int, numRestoreItems int,
) { ) {
t.Logf( t.Logf(
"Restoring collections to %s for resourceOwners(s) %v\n", "Restoring collections to %s for resourceOwners(s) %v\n",
config.dest.ContainerName, config.Dest.ContainerName,
config.resourceOwners) config.ResourceOwners)
start := time.Now() start := time.Now()
restoreGC := loadConnector(ctx, t, config.resource) restoreGC := loadConnector(ctx, t, config.Resource)
restoreSel := getSelectorWith(t, config.service, config.resourceOwners, true) restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true)
deets, err := restoreGC.ConsumeRestoreCollections( deets, err := restoreGC.ConsumeRestoreCollections(
ctx, ctx,
backupVersion, backupVersion,
config.acct, config.Acct,
restoreSel, restoreSel,
config.dest, config.Dest,
config.opts, config.Opts,
collections, collections,
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -490,30 +453,30 @@ func runRestore(
func runBackupAndCompare( func runBackupAndCompare(
t *testing.T, t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
config configInfo, config ConfigInfo,
expectedData map[string]map[string][]byte, expectedData map[string]map[string][]byte,
totalItems int, totalItems int,
totalKopiaItems int, totalKopiaItems int,
inputCollections []colInfo, inputCollections []ColInfo,
) { ) {
t.Helper() t.Helper()
// Run a backup and compare its output with what we put in. // Run a backup and compare its output with what we put in.
cats := make(map[path.CategoryType]struct{}, len(inputCollections)) cats := make(map[path.CategoryType]struct{}, len(inputCollections))
for _, c := range inputCollections { for _, c := range inputCollections {
cats[c.category] = struct{}{} cats[c.Category] = struct{}{}
} }
var ( var (
expectedDests = make([]destAndCats, 0, len(config.resourceOwners)) expectedDests = make([]destAndCats, 0, len(config.ResourceOwners))
idToName = map[string]string{} idToName = map[string]string{}
nameToID = map[string]string{} nameToID = map[string]string{}
) )
for _, ro := range config.resourceOwners { for _, ro := range config.ResourceOwners {
expectedDests = append(expectedDests, destAndCats{ expectedDests = append(expectedDests, destAndCats{
resourceOwner: ro, resourceOwner: ro,
dest: config.dest.ContainerName, dest: config.Dest.ContainerName,
cats: cats, cats: cats,
}) })
@ -521,10 +484,10 @@ func runBackupAndCompare(
nameToID[ro] = ro nameToID[ro] = ro
} }
backupGC := loadConnector(ctx, t, config.resource) backupGC := loadConnector(ctx, t, config.Resource)
backupGC.IDNameLookup = inMock.NewCache(idToName, nameToID) backupGC.IDNameLookup = inMock.NewCache(idToName, nameToID)
backupSel := backupSelectorForExpected(t, config.service, expectedDests) backupSel := backupSelectorForExpected(t, config.Service, expectedDests)
t.Logf("Selective backup of %s\n", backupSel) t.Logf("Selective backup of %s\n", backupSel)
start := time.Now() start := time.Now()
@ -534,7 +497,7 @@ func runBackupAndCompare(
backupSel, backupSel,
nil, nil,
version.NoBackup, version.NoBackup,
config.opts, config.Opts,
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// No excludes yet because this isn't an incremental backup. // No excludes yet because this isn't an incremental backup.
@ -550,8 +513,8 @@ func runBackupAndCompare(
totalKopiaItems, totalKopiaItems,
expectedData, expectedData,
dcs, dcs,
config.dest, config.Dest,
config.opts.RestorePermissions) config.Opts.RestorePermissions)
status := backupGC.Wait() status := backupGC.Wait()
@ -572,17 +535,17 @@ func runRestoreBackupTest(
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
config := configInfo{ config := ConfigInfo{
acct: acct, Acct: acct,
opts: opts, Opts: opts,
resource: test.resource, Resource: test.resource,
service: test.service, Service: test.service,
tenant: tenant, Tenant: tenant,
resourceOwners: resourceOwners, ResourceOwners: resourceOwners,
dest: tester.DefaultTestRestoreDestination(), Dest: tester.DefaultTestRestoreDestination(),
} }
totalItems, totalKopiaItems, collections, expectedData := getCollectionsAndExpected( totalItems, totalKopiaItems, collections, expectedData := GetCollectionsAndExpected(
t, t,
config, config,
test.collections, test.collections,
@ -618,17 +581,17 @@ func runRestoreTestWithVerion(
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
config := configInfo{ config := ConfigInfo{
acct: acct, Acct: acct,
opts: opts, Opts: opts,
resource: test.resource, Resource: test.resource,
service: test.service, Service: test.service,
tenant: tenant, Tenant: tenant,
resourceOwners: resourceOwners, ResourceOwners: resourceOwners,
dest: tester.DefaultTestRestoreDestination(), Dest: tester.DefaultTestRestoreDestination(),
} }
totalItems, _, collections, _ := getCollectionsAndExpected( totalItems, _, collections, _ := GetCollectionsAndExpected(
t, t,
config, config,
test.collectionsPrevious, test.collectionsPrevious,
@ -657,17 +620,17 @@ func runRestoreBackupTestVersions(
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
config := configInfo{ config := ConfigInfo{
acct: acct, Acct: acct,
opts: opts, Opts: opts,
resource: test.resource, Resource: test.resource,
service: test.service, Service: test.service,
tenant: tenant, Tenant: tenant,
resourceOwners: resourceOwners, ResourceOwners: resourceOwners,
dest: tester.DefaultTestRestoreDestination(), Dest: tester.DefaultTestRestoreDestination(),
} }
totalItems, _, collections, _ := getCollectionsAndExpected( totalItems, _, collections, _ := GetCollectionsAndExpected(
t, t,
config, config,
test.collectionsPrevious, test.collectionsPrevious,
@ -682,7 +645,7 @@ func runRestoreBackupTestVersions(
totalItems) totalItems)
// Get expected output for new version. // Get expected output for new version.
totalItems, totalKopiaItems, _, expectedData := getCollectionsAndExpected( totalItems, totalKopiaItems, _, expectedData := GetCollectionsAndExpected(
t, t,
config, config,
test.collectionsLatest, test.collectionsLatest,
@ -707,11 +670,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
name: "EmailsWithAttachments", name: "EmailsWithAttachments",
service: path.ExchangeService, service: path.ExchangeService,
resource: Users, resource: Users,
collections: []colInfo{ collections: []ColInfo{
{ {
pathElements: []string{"Inbox"}, PathElements: []string{"Inbox"},
category: path.EmailCategory, Category: path.EmailCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID", name: "someencodeditemID",
data: exchMock.MessageWithDirectAttachment( data: exchMock.MessageWithDirectAttachment(
@ -734,11 +697,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleEmailsMultipleFolders", name: "MultipleEmailsMultipleFolders",
service: path.ExchangeService, service: path.ExchangeService,
resource: Users, resource: Users,
collections: []colInfo{ collections: []ColInfo{
{ {
pathElements: []string{"Inbox"}, PathElements: []string{"Inbox"},
category: path.EmailCategory, Category: path.EmailCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID", name: "someencodeditemID",
data: exchMock.MessageWithBodyBytes( data: exchMock.MessageWithBodyBytes(
@ -751,9 +714,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
}, },
}, },
{ {
pathElements: []string{"Work"}, PathElements: []string{"Work"},
category: path.EmailCategory, Category: path.EmailCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID2", name: "someencodeditemID2",
data: exchMock.MessageWithBodyBytes( data: exchMock.MessageWithBodyBytes(
@ -775,9 +738,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
}, },
}, },
{ {
pathElements: []string{"Work", "Inbox"}, PathElements: []string{"Work", "Inbox"},
category: path.EmailCategory, Category: path.EmailCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID4", name: "someencodeditemID4",
data: exchMock.MessageWithBodyBytes( data: exchMock.MessageWithBodyBytes(
@ -790,9 +753,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
}, },
}, },
{ {
pathElements: []string{"Work", "Inbox", "Work"}, PathElements: []string{"Work", "Inbox", "Work"},
category: path.EmailCategory, Category: path.EmailCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID5", name: "someencodeditemID5",
data: exchMock.MessageWithBodyBytes( data: exchMock.MessageWithBodyBytes(
@ -810,11 +773,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleContactsSingleFolder", name: "MultipleContactsSingleFolder",
service: path.ExchangeService, service: path.ExchangeService,
resource: Users, resource: Users,
collections: []colInfo{ collections: []ColInfo{
{ {
pathElements: []string{"Contacts"}, PathElements: []string{"Contacts"},
category: path.ContactsCategory, Category: path.ContactsCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID", name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"), data: exchMock.ContactBytes("Ghimley"),
@ -838,11 +801,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleContactsMultipleFolders", name: "MultipleContactsMultipleFolders",
service: path.ExchangeService, service: path.ExchangeService,
resource: Users, resource: Users,
collections: []colInfo{ collections: []ColInfo{
{ {
pathElements: []string{"Work"}, PathElements: []string{"Work"},
category: path.ContactsCategory, Category: path.ContactsCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID", name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"), data: exchMock.ContactBytes("Ghimley"),
@ -861,9 +824,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
}, },
}, },
{ {
pathElements: []string{"Personal"}, PathElements: []string{"Personal"},
category: path.ContactsCategory, Category: path.ContactsCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID4", name: "someencodeditemID4",
data: exchMock.ContactBytes("Argon"), data: exchMock.ContactBytes("Argon"),
@ -973,11 +936,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
name: "Contacts", name: "Contacts",
service: path.ExchangeService, service: path.ExchangeService,
resource: Users, resource: Users,
collections: []colInfo{ collections: []ColInfo{
{ {
pathElements: []string{"Work"}, PathElements: []string{"Work"},
category: path.ContactsCategory, Category: path.ContactsCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID", name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"), data: exchMock.ContactBytes("Ghimley"),
@ -986,9 +949,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
}, },
}, },
{ {
pathElements: []string{"Personal"}, PathElements: []string{"Personal"},
category: path.ContactsCategory, Category: path.ContactsCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "someencodeditemID2", name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"), data: exchMock.ContactBytes("Irgot"),
@ -1014,9 +977,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
// }, // },
// }, // },
// { // {
// pathElements: []string{"Personal"}, // PathElements: []string{"Personal"},
// category: path.EventsCategory, // Category: path.EventsCategory,
// items: []itemInfo{ // Items: []ItemInfo{
// { // {
// name: "someencodeditemID2", // name: "someencodeditemID2",
// data: exchMock.EventWithSubjectBytes("Irgot"), // data: exchMock.EventWithSubjectBytes("Irgot"),
@ -1047,7 +1010,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
resourceOwner: suite.user, resourceOwner: suite.user,
dest: dest.ContainerName, dest: dest.ContainerName,
cats: map[path.CategoryType]struct{}{ cats: map[path.CategoryType]struct{}{
collection.category: {}, collection.Category: {},
}, },
}) })
@ -1057,7 +1020,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
suite.connector.tenant, suite.connector.tenant,
suite.user, suite.user,
dest, dest,
[]colInfo{collection}, []ColInfo{collection},
version.Backup, version.Backup,
) )
allItems += totalItems allItems += totalItems
@ -1153,11 +1116,11 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup_largeMailAttac
name: "EmailsWithLargeAttachments", name: "EmailsWithLargeAttachments",
service: path.ExchangeService, service: path.ExchangeService,
resource: Users, resource: Users,
collections: []colInfo{ collections: []ColInfo{
{ {
pathElements: []string{"Inbox"}, PathElements: []string{"Inbox"},
category: path.EmailCategory, Category: path.EmailCategory,
items: []itemInfo{ Items: []ItemInfo{
{ {
name: "35mbAttachment", name: "35mbAttachment",
data: exchMock.MessageWithSizedAttachment(subjectText, 35), data: exchMock.MessageWithSizedAttachment(subjectText, 35),

187
src/internal/connector/graph_connector_test_helper.go Normal file
View File

@ -0,0 +1,187 @@
package connector
import (
"bytes"
"context"
"io"
"testing"
"github.com/alcionai/clues"
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/stretchr/testify/require"
)
type ColInfo struct {
// Elements (in order) for the path representing this collection. Should
// only contain elements after the prefix that corso uses for the path. For
// example, a collection for the Inbox folder in exchange mail would just be
// "Inbox".
PathElements []string
Category path.CategoryType
Items []ItemInfo
// auxItems are items that can be retrieved with Fetch but won't be returned
// by Items(). These files do not directly participate in comparisosn at the
// end of a test.
AuxItems []ItemInfo
}
type ItemInfo struct {
// lookupKey is a string that can be used to find this data from a set of
// other data in the same collection. This key should be something that will
// be the same before and after restoring the item in M365 and may not be
// the M365 ID. When restoring items out of place, the item is assigned a
// new ID making it unsuitable for a lookup key.
lookupKey string
name string
data []byte
}
type ConfigInfo struct {
Acct account.Account
Opts control.Options
Resource Resource
Service path.ServiceType
Tenant string
ResourceOwners []string
Dest control.RestoreDestination
}
func mustToDataLayerPath(
t *testing.T,
service path.ServiceType,
tenant, resourceOwner string,
category path.CategoryType,
elements []string,
isItem bool,
) path.Path {
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
require.NoError(t, err, clues.ToCore(err))
return res
}
// backupOutputPathFromRestore returns a path.Path denoting the location in
// kopia the data will be placed at. The location is a data-type specific
// combination of the location the data was recently restored to and where the
// data was originally in the hierarchy.
func backupOutputPathFromRestore(
t *testing.T,
restoreDest control.RestoreDestination,
inputPath path.Path,
) path.Path {
base := []string{restoreDest.ContainerName}
// OneDrive has leading information like the drive ID.
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService {
folders := inputPath.Folders()
base = append(append([]string{}, folders[:3]...), restoreDest.ContainerName)
if len(folders) > 3 {
base = append(base, folders[3:]...)
}
}
if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory {
base = append(base, inputPath.Folders()...)
}
return mustToDataLayerPath(
t,
inputPath.Service(),
inputPath.Tenant(),
inputPath.ResourceOwner(),
inputPath.Category(),
base,
false,
)
}
// TODO(ashmrtn): Make this an actual mock class that can be used in other
// packages.
type mockRestoreCollection struct {
data.Collection
auxItems map[string]data.Stream
}
func (rc mockRestoreCollection) Fetch(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.auxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}
func collectionsForInfo(
t *testing.T,
service path.ServiceType,
tenant, user string,
dest control.RestoreDestination,
allInfo []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
var (
collections = make([]data.RestoreCollection, 0, len(allInfo))
expectedData = make(map[string]map[string][]byte, len(allInfo))
totalItems = 0
kopiaEntries = 0
)
for _, info := range allInfo {
pth := mustToDataLayerPath(
t,
service,
tenant,
user,
info.Category,
info.PathElements,
false)
mc := exchMock.NewCollection(pth, pth, len(info.Items))
baseDestPath := backupOutputPathFromRestore(t, dest, pth)
baseExpected := expectedData[baseDestPath.String()]
if baseExpected == nil {
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items))
baseExpected = expectedData[baseDestPath.String()]
}
for i := 0; i < len(info.Items); i++ {
mc.Names[i] = info.Items[i].name
mc.Data[i] = info.Items[i].data
baseExpected[info.Items[i].lookupKey] = info.Items[i].data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].name) {
continue
}
totalItems++
}
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.AuxItems {
c.auxItems[aux.name] = &exchMock.Data{
ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.Items)
}
return totalItems, kopiaEntries, collections, expectedData
}