larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

final cleanup for renaming/movement (#3612)

Browse Source 
This last step in the cleanup has two goals: first to minimize the number of arbitrarily named and located files so that code is better condensed and co-located.  This is all just file renaming and a minor amount of code copy-pasting.

The second is to create a centralized package to own the ColInfo type structs that we use to both stub out test data, and also generate factory data for cmds.  The current ownership is haphazard, and while this movement is a little more condensed, it's still jumping through some weird hoops to get things to work.  Treat it as one good step forward, and we'll have to return to polish it another time.  At least it'll be separated from the m365 folder at large this way, and more easily identified as supporting design rather than production usage.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* closes #1996

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-06-14 10:50:17 -06:00 committed by GitHub
parent 466698f096
commit bd0f6f9769
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 705 additions and 737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/cmd/factory/impl/common.go
View File

@ -18,7 +18,9 @@ import (
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365" "github.com/alcionai/corso/src/internal/m365"
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
@ -84,14 +86,14 @@ func generateAndRestoreItems(
items: items, items: items,
}} }}
dest := control.DefaultRestoreConfig(dttm.SafeForTesting) restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
dest.Location = destFldr restoreCfg.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.Location) print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
dataColls, err := buildCollections( dataColls, err := buildCollections(
service, service,
tenantID, userID, tenantID, userID,
dest, restoreCfg,
collections) collections)
if err != nil { if err != nil {
return nil, err return nil, err
@ -99,7 +101,7 @@ func generateAndRestoreItems(
print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, dataColls, errs) return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs)
} }
// ------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------
@ -108,7 +110,7 @@ func generateAndRestoreItems(
func getControllerAndVerifyResourceOwner( func getControllerAndVerifyResourceOwner(
ctx context.Context, ctx context.Context,
rc resource.Category, resourceCat resource.Category,
resourceOwner string, resourceOwner string,
) ( ) (
*m365.Controller, *m365.Controller,
@ -133,7 +135,7 @@ func getControllerAndVerifyResourceOwner(
return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details") return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details")
} }
ctrl, err := m365.NewController(ctx, acct, rc) ctrl, err := m365.NewController(ctx, acct, resourceCat)
if err != nil { if err != nil {
return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
} }
@ -164,7 +166,7 @@ type collection struct {
func buildCollections( func buildCollections(
service path.ServiceType, service path.ServiceType,
tenant, user string, tenant, user string,
dest control.RestoreConfig, restoreCfg control.RestoreConfig,
colls []collection, colls []collection,
) ([]data.RestoreCollection, error) { ) ([]data.RestoreCollection, error) {
collections := make([]data.RestoreCollection, 0, len(colls)) collections := make([]data.RestoreCollection, 0, len(colls))
@ -225,9 +227,9 @@ func generateAndRestoreDriveItems(
ctx, flush := tester.NewContext(nil) ctx, flush := tester.NewContext(nil)
defer flush() defer flush()
dest := control.DefaultRestoreConfig(dttm.SafeForTesting) restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
dest.Location = destFldr restoreCfg.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.Location) print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
var driveID string var driveID string
@ -249,7 +251,7 @@ func generateAndRestoreDriveItems(
} }
var ( var (
cols []m365.OnedriveColInfo cols []odStub.ColInfo
rootPath = []string{"drives", driveID, "root:"} rootPath = []string{"drives", driveID, "root:"}
folderAPath = []string{"drives", driveID, "root:", folderAName} folderAPath = []string{"drives", driveID, "root:", folderAName}
@ -263,15 +265,15 @@ func generateAndRestoreDriveItems(
) )
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
col := []m365.OnedriveColInfo{ col := []odStub.ColInfo{
// basic folder and file creation // basic folder and file creation
{ {
PathElements: rootPath, PathElements: rootPath,
Files: []m365.ItemData{ Files: []odStub.ItemData{
{ {
Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
Data: fileAData, Data: fileAData,
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
@ -282,13 +284,13 @@ func generateAndRestoreDriveItems(
Data: fileBData, Data: fileBData,
}, },
}, },
Folders: []m365.ItemData{ Folders: []odStub.ItemData{
{ {
Name: folderBName, Name: folderBName,
}, },
{ {
Name: folderAName, Name: folderAName,
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -296,7 +298,7 @@ func generateAndRestoreDriveItems(
}, },
{ {
Name: folderCName, Name: folderCName,
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -308,18 +310,18 @@ func generateAndRestoreDriveItems(
// a folder that has permissions with an item in the folder with // a folder that has permissions with an item in the folder with
// the different permissions. // the different permissions.
PathElements: folderAPath, PathElements: folderAPath,
Files: []m365.ItemData{ Files: []odStub.ItemData{
{ {
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
Data: fileEData, Data: fileEData,
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -329,13 +331,13 @@ func generateAndRestoreDriveItems(
// a folder that has permissions with an item in the folder with // a folder that has permissions with an item in the folder with
// no permissions. // no permissions.
PathElements: folderCPath, PathElements: folderCPath,
Files: []m365.ItemData{ Files: []odStub.ItemData{
{ {
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
Data: fileAData, Data: fileAData,
}, },
}, },
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -343,23 +345,23 @@ func generateAndRestoreDriveItems(
}, },
{ {
PathElements: folderBPath, PathElements: folderBPath,
Files: []m365.ItemData{ Files: []odStub.ItemData{
{ {
// restoring a file in a non-root folder that doesn't inherit // restoring a file in a non-root folder that doesn't inherit
// permissions. // permissions.
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime), Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
Data: fileBData, Data: fileBData,
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
Folders: []m365.ItemData{ Folders: []odStub.ItemData{
{ {
Name: folderAName, Name: folderAName,
Perms: m365.PermData{ Perms: odStub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -372,7 +374,7 @@ func generateAndRestoreDriveItems(
cols = append(cols, col...) cols = append(cols, col...)
} }
input, err := m365.DataForInfo(service, cols, version.Backup) input, err := odStub.DataForInfo(service, cols, version.Backup)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -389,7 +391,7 @@ func generateAndRestoreDriveItems(
ToggleFeatures: control.Toggles{}, ToggleFeatures: control.Toggles{},
} }
config := m365.ConfigInfo{ config := m365Stub.ConfigInfo{
Opts: opts, Opts: opts,
Resource: resource.Users, Resource: resource.Users,
Service: service, Service: service,
@ -398,7 +400,7 @@ func generateAndRestoreDriveItems(
RestoreCfg: tester.DefaultTestRestoreConfig(""), RestoreCfg: tester.DefaultTestRestoreConfig(""),
} }
_, _, collections, _, err := m365.GetCollectionsAndExpected( _, _, collections, _, err := m365Stub.GetCollectionsAndExpected(
config, config,
input, input,
version.Backup) version.Backup)
@ -406,5 +408,5 @@ func generateAndRestoreDriveItems(
return nil, err return nil, err
} }
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, collections, errs) return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs)
} }

12
src/internal/m365/backup_test.go
View File

@ -66,7 +66,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() {
selUsers := []string{suite.user} selUsers := []string{suite.user}
ctrl := loadController(ctx, suite.T(), resource.Users) ctrl := newController(ctx, suite.T(), resource.Users)
tests := []struct { tests := []struct {
name string name string
getSelector func(t *testing.T) selectors.Selector getSelector func(t *testing.T) selectors.Selector
@ -167,7 +167,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner()
defer flush() defer flush()
owners := []string{"snuffleupagus"} owners := []string{"snuffleupagus"}
ctrl := loadController(ctx, suite.T(), resource.Users) ctrl := newController(ctx, suite.T(), resource.Users)
tests := []struct { tests := []struct {
name string name string
getSelector func(t *testing.T) selectors.Selector getSelector func(t *testing.T) selectors.Selector
@ -253,7 +253,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
defer flush() defer flush()
selSites := []string{suite.site} selSites := []string{suite.site}
ctrl := loadController(ctx, suite.T(), resource.Sites) ctrl := newController(ctx, suite.T(), resource.Sites)
tests := []struct { tests := []struct {
name string name string
expected int expected int
@ -348,7 +348,7 @@ func (suite *SPCollectionIntgSuite) SetupSuite() {
ctx, flush := tester.NewContext(suite.T()) ctx, flush := tester.NewContext(suite.T())
defer flush() defer flush()
suite.connector = loadController(ctx, suite.T(), resource.Sites) suite.connector = newController(ctx, suite.T(), resource.Sites)
suite.user = tester.M365UserID(suite.T()) suite.user = tester.M365UserID(suite.T())
tester.LogTimeOfTest(suite.T()) tester.LogTimeOfTest(suite.T())
@ -362,7 +362,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
var ( var (
siteID = tester.M365SiteID(t) siteID = tester.M365SiteID(t)
ctrl = loadController(ctx, t, resource.Sites) ctrl = newController(ctx, t, resource.Sites)
siteIDs = []string{siteID} siteIDs = []string{siteID}
) )
@ -409,7 +409,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
var ( var (
siteID = tester.M365SiteID(t) siteID = tester.M365SiteID(t)
ctrl = loadController(ctx, t, resource.Sites) ctrl = newController(ctx, t, resource.Sites)
siteIDs = []string{siteID} siteIDs = []string{siteID}
) )

349
src/internal/m365/controller_test.go
View File

@ -17,6 +17,7 @@ import (
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
"github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
@ -24,6 +25,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
) )
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -280,7 +282,7 @@ func (suite *ControllerIntegrationSuite) SetupSuite() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
suite.ctrl = loadController(ctx, t, resource.Users) suite.ctrl = newController(ctx, t, resource.Users)
suite.user = tester.M365UserID(t) suite.user = tester.M365UserID(t)
suite.secondaryUser = tester.SecondaryM365UserID(t) suite.secondaryUser = tester.SecondaryM365UserID(t)
@ -407,7 +409,7 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() {
func runRestore( func runRestore(
t *testing.T, t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
config ConfigInfo, config stub.ConfigInfo,
backupVersion int, backupVersion int,
collections []data.RestoreCollection, collections []data.RestoreCollection,
numRestoreItems int, numRestoreItems int,
@ -419,7 +421,7 @@ func runRestore(
start := time.Now() start := time.Now()
restoreCtrl := loadController(ctx, t, config.Resource) restoreCtrl := newController(ctx, t, config.Resource)
restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true) restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true)
deets, err := restoreCtrl.ConsumeRestoreCollections( deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx, ctx,
@ -450,11 +452,11 @@ func runRestore(
func runBackupAndCompare( func runBackupAndCompare(
t *testing.T, t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
config ConfigInfo, config stub.ConfigInfo,
expectedData map[string]map[string][]byte, expectedData map[string]map[string][]byte,
totalItems int, totalItems int,
totalKopiaItems int, totalKopiaItems int,
inputCollections []ColInfo, inputCollections []stub.ColInfo,
) { ) {
t.Helper() t.Helper()
@ -481,7 +483,7 @@ func runBackupAndCompare(
nameToID[ro] = ro nameToID[ro] = ro
} }
backupCtrl := loadController(ctx, t, config.Resource) backupCtrl := newController(ctx, t, config.Resource)
backupCtrl.IDNameLookup = inMock.NewCache(idToName, nameToID) backupCtrl.IDNameLookup = inMock.NewCache(idToName, nameToID)
backupSel := backupSelectorForExpected(t, config.Service, expectedDests) backupSel := backupSelectorForExpected(t, config.Service, expectedDests)
@ -531,7 +533,7 @@ func runRestoreBackupTest(
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
config := ConfigInfo{ config := stub.ConfigInfo{
Opts: opts, Opts: opts,
Resource: test.resourceCat, Resource: test.resourceCat,
Service: test.service, Service: test.service,
@ -540,7 +542,7 @@ func runRestoreBackupTest(
RestoreCfg: tester.DefaultTestRestoreConfig(""), RestoreCfg: tester.DefaultTestRestoreConfig(""),
} }
totalItems, totalKopiaItems, collections, expectedData, err := GetCollectionsAndExpected( totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected(
config, config,
test.collections, test.collections,
version.Backup) version.Backup)
@ -576,16 +578,16 @@ func runRestoreTestWithVersion(
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
config := ConfigInfo{ config := stub.ConfigInfo{
Opts: opts, Opts: opts,
Resource: test.resource, Resource: test.resourceCat,
Service: test.service, Service: test.service,
Tenant: tenant, Tenant: tenant,
ResourceOwners: resourceOwners, ResourceOwners: resourceOwners,
RestoreCfg: tester.DefaultTestRestoreConfig(""), RestoreCfg: tester.DefaultTestRestoreConfig(""),
} }
totalItems, _, collections, _, err := GetCollectionsAndExpected( totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
config, config,
test.collectionsPrevious, test.collectionsPrevious,
test.backupVersion) test.backupVersion)
@ -613,16 +615,16 @@ func runRestoreBackupTestVersions(
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
config := ConfigInfo{ config := stub.ConfigInfo{
Opts: opts, Opts: opts,
Resource: test.resource, Resource: test.resourceCat,
Service: test.service, Service: test.service,
Tenant: tenant, Tenant: tenant,
ResourceOwners: resourceOwners, ResourceOwners: resourceOwners,
RestoreCfg: tester.DefaultTestRestoreConfig(""), RestoreCfg: tester.DefaultTestRestoreConfig(""),
} }
totalItems, _, collections, _, err := GetCollectionsAndExpected( totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
config, config,
test.collectionsPrevious, test.collectionsPrevious,
test.backupVersion) test.backupVersion)
@ -637,7 +639,7 @@ func runRestoreBackupTestVersions(
totalItems) totalItems)
// Get expected output for new version. // Get expected output for new version.
totalItems, totalKopiaItems, _, expectedData, err := GetCollectionsAndExpected( totalItems, totalKopiaItems, _, expectedData, err := stub.GetCollectionsAndExpected(
config, config,
test.collectionsLatest, test.collectionsLatest,
version.Backup) version.Backup)
@ -662,24 +664,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "EmailsWithAttachments", name: "EmailsWithAttachments",
service: path.ExchangeService, service: path.ExchangeService,
resourceCat: resource.Users, resourceCat: resource.Users,
collections: []ColInfo{ collections: []stub.ColInfo{
{ {
PathElements: []string{"Inbox"}, PathElements: []string{"Inbox"},
Category: path.EmailCategory, Category: path.EmailCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID", Name: "someencodeditemID",
data: exchMock.MessageWithDirectAttachment( Data: exchMock.MessageWithDirectAttachment(
subjectText + "-1", subjectText + "-1",
), ),
lookupKey: subjectText + "-1", LookupKey: subjectText + "-1",
}, },
{ {
name: "someencodeditemID2", Name: "someencodeditemID2",
data: exchMock.MessageWithTwoAttachments( Data: exchMock.MessageWithTwoAttachments(
subjectText + "-2", subjectText + "-2",
), ),
lookupKey: subjectText + "-2", LookupKey: subjectText + "-2",
}, },
}, },
}, },
@ -689,73 +691,73 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleEmailsMultipleFolders", name: "MultipleEmailsMultipleFolders",
service: path.ExchangeService, service: path.ExchangeService,
resourceCat: resource.Users, resourceCat: resource.Users,
collections: []ColInfo{ collections: []stub.ColInfo{
{ {
PathElements: []string{"Inbox"}, PathElements: []string{"Inbox"},
Category: path.EmailCategory, Category: path.EmailCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID", Name: "someencodeditemID",
data: exchMock.MessageWithBodyBytes( Data: exchMock.MessageWithBodyBytes(
subjectText+"-1", subjectText+"-1",
bodyText+" 1.", bodyText+" 1.",
bodyText+" 1.", bodyText+" 1.",
), ),
lookupKey: subjectText + "-1", LookupKey: subjectText + "-1",
}, },
}, },
}, },
{ {
PathElements: []string{"Work"}, PathElements: []string{"Work"},
Category: path.EmailCategory, Category: path.EmailCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID2", Name: "someencodeditemID2",
data: exchMock.MessageWithBodyBytes( Data: exchMock.MessageWithBodyBytes(
subjectText+"-2", subjectText+"-2",
bodyText+" 2.", bodyText+" 2.",
bodyText+" 2.", bodyText+" 2.",
), ),
lookupKey: subjectText + "-2", LookupKey: subjectText + "-2",
}, },
{ {
name: "someencodeditemID3", Name: "someencodeditemID3",
data: exchMock.MessageWithBodyBytes( Data: exchMock.MessageWithBodyBytes(
subjectText+"-3", subjectText+"-3",
bodyText+" 3.", bodyText+" 3.",
bodyText+" 3.", bodyText+" 3.",
), ),
lookupKey: subjectText + "-3", LookupKey: subjectText + "-3",
}, },
}, },
}, },
{ {
PathElements: []string{"Work", "Inbox"}, PathElements: []string{"Work", "Inbox"},
Category: path.EmailCategory, Category: path.EmailCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID4", Name: "someencodeditemID4",
data: exchMock.MessageWithBodyBytes( Data: exchMock.MessageWithBodyBytes(
subjectText+"-4", subjectText+"-4",
bodyText+" 4.", bodyText+" 4.",
bodyText+" 4.", bodyText+" 4.",
), ),
lookupKey: subjectText + "-4", LookupKey: subjectText + "-4",
}, },
}, },
}, },
{ {
PathElements: []string{"Work", "Inbox", "Work"}, PathElements: []string{"Work", "Inbox", "Work"},
Category: path.EmailCategory, Category: path.EmailCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID5", Name: "someencodeditemID5",
data: exchMock.MessageWithBodyBytes( Data: exchMock.MessageWithBodyBytes(
subjectText+"-5", subjectText+"-5",
bodyText+" 5.", bodyText+" 5.",
bodyText+" 5.", bodyText+" 5.",
), ),
lookupKey: subjectText + "-5", LookupKey: subjectText + "-5",
}, },
}, },
}, },
@ -765,25 +767,25 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleContactsSingleFolder", name: "MultipleContactsSingleFolder",
service: path.ExchangeService, service: path.ExchangeService,
resourceCat: resource.Users, resourceCat: resource.Users,
collections: []ColInfo{ collections: []stub.ColInfo{
{ {
PathElements: []string{"Contacts"}, PathElements: []string{"Contacts"},
Category: path.ContactsCategory, Category: path.ContactsCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID", Name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"), Data: exchMock.ContactBytes("Ghimley"),
lookupKey: "Ghimley", LookupKey: "Ghimley",
}, },
{ {
name: "someencodeditemID2", Name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"), Data: exchMock.ContactBytes("Irgot"),
lookupKey: "Irgot", LookupKey: "Irgot",
}, },
{ {
name: "someencodeditemID3", Name: "someencodeditemID3",
data: exchMock.ContactBytes("Jannes"), Data: exchMock.ContactBytes("Jannes"),
lookupKey: "Jannes", LookupKey: "Jannes",
}, },
}, },
}, },
@ -793,41 +795,41 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleContactsMultipleFolders", name: "MultipleContactsMultipleFolders",
service: path.ExchangeService, service: path.ExchangeService,
resourceCat: resource.Users, resourceCat: resource.Users,
collections: []ColInfo{ collections: []stub.ColInfo{
{ {
PathElements: []string{"Work"}, PathElements: []string{"Work"},
Category: path.ContactsCategory, Category: path.ContactsCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID", Name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"), Data: exchMock.ContactBytes("Ghimley"),
lookupKey: "Ghimley", LookupKey: "Ghimley",
}, },
{ {
name: "someencodeditemID2", Name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"), Data: exchMock.ContactBytes("Irgot"),
lookupKey: "Irgot", LookupKey: "Irgot",
}, },
{ {
name: "someencodeditemID3", Name: "someencodeditemID3",
data: exchMock.ContactBytes("Jannes"), Data: exchMock.ContactBytes("Jannes"),
lookupKey: "Jannes", LookupKey: "Jannes",
}, },
}, },
}, },
{ {
PathElements: []string{"Personal"}, PathElements: []string{"Personal"},
Category: path.ContactsCategory, Category: path.ContactsCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID4", Name: "someencodeditemID4",
data: exchMock.ContactBytes("Argon"), Data: exchMock.ContactBytes("Argon"),
lookupKey: "Argon", LookupKey: "Argon",
}, },
{ {
name: "someencodeditemID5", Name: "someencodeditemID5",
data: exchMock.ContactBytes("Bernard"), Data: exchMock.ContactBytes("Bernard"),
lookupKey: "Bernard", LookupKey: "Bernard",
}, },
}, },
}, },
@ -926,26 +928,26 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
name: "Contacts", name: "Contacts",
service: path.ExchangeService, service: path.ExchangeService,
resourceCat: resource.Users, resourceCat: resource.Users,
collections: []ColInfo{ collections: []stub.ColInfo{
{ {
PathElements: []string{"Work"}, PathElements: []string{"Work"},
Category: path.ContactsCategory, Category: path.ContactsCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID", Name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"), Data: exchMock.ContactBytes("Ghimley"),
lookupKey: "Ghimley", LookupKey: "Ghimley",
}, },
}, },
}, },
{ {
PathElements: []string{"Personal"}, PathElements: []string{"Personal"},
Category: path.ContactsCategory, Category: path.ContactsCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "someencodeditemID2", Name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"), Data: exchMock.ContactBytes("Irgot"),
lookupKey: "Irgot", LookupKey: "Irgot",
}, },
}, },
}, },
@ -1004,12 +1006,12 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
}, },
}) })
totalItems, _, collections, expectedData, err := collectionsForInfo( totalItems, _, collections, expectedData, err := stub.CollectionsForInfo(
test.service, test.service,
suite.ctrl.tenant, suite.ctrl.tenant,
suite.user, suite.user,
restoreCfg, restoreCfg,
[]ColInfo{collection}, []stub.ColInfo{collection},
version.Backup, version.Backup,
) )
require.NoError(t, err) require.NoError(t, err)
@ -1027,7 +1029,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
restoreCfg.Location, restoreCfg.Location,
) )
restoreCtrl := loadController(ctx, t, test.resourceCat) restoreCtrl := newController(ctx, t, test.resourceCat)
deets, err := restoreCtrl.ConsumeRestoreCollections( deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx, ctx,
version.Backup, version.Backup,
@ -1057,7 +1059,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
// Run a backup and compare its output with what we put in. // Run a backup and compare its output with what we put in.
backupCtrl := loadController(ctx, t, test.resourceCat) backupCtrl := newController(ctx, t, test.resourceCat)
backupSel := backupSelectorForExpected(t, test.service, expectedDests) backupSel := backupSelectorForExpected(t, test.service, expectedDests)
t.Log("Selective backup of", backupSel) t.Log("Selective backup of", backupSel)
@ -1079,7 +1081,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
t.Log("Backup enumeration complete") t.Log("Backup enumeration complete")
ci := ConfigInfo{ ci := stub.ConfigInfo{
Opts: control.Options{RestorePermissions: true}, Opts: control.Options{RestorePermissions: true},
// Alright to be empty, needed for OneDrive. // Alright to be empty, needed for OneDrive.
RestoreCfg: control.RestoreConfig{}, RestoreCfg: control.RestoreConfig{},
@ -1105,15 +1107,15 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
name: "EmailsWithLargeAttachments", name: "EmailsWithLargeAttachments",
service: path.ExchangeService, service: path.ExchangeService,
resourceCat: resource.Users, resourceCat: resource.Users,
collections: []ColInfo{ collections: []stub.ColInfo{
{ {
PathElements: []string{"Inbox"}, PathElements: []string{"Inbox"},
Category: path.EmailCategory, Category: path.EmailCategory,
Items: []ItemInfo{ Items: []stub.ItemInfo{
{ {
name: "35mbAttachment", Name: "35mbAttachment",
data: exchMock.MessageWithSizedAttachment(subjectText, 35), Data: exchMock.MessageWithSizedAttachment(subjectText, 35),
lookupKey: subjectText, LookupKey: subjectText,
}, },
}, },
}, },
@ -1206,7 +1208,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
defer flush() defer flush()
var ( var (
backupCtrl = loadController(ctx, t, test.resourceCat) backupCtrl = newController(ctx, t, test.resourceCat)
backupSel = test.selectorFunc(t) backupSel = test.selectorFunc(t)
errs = fault.New(true) errs = fault.New(true)
start = time.Now() start = time.Now()
@ -1270,3 +1272,166 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
}) })
} }
} }
type DisconnectedUnitSuite struct {
tester.Suite
}
func TestDisconnectedUnitSuite(t *testing.T) {
s := &DisconnectedUnitSuite{
Suite: tester.NewUnitSuite(t),
}
suite.Run(t, s)
}
func statusTestTask(
t *testing.T,
ctrl *Controller,
objects, success, folder int,
) {
ctx, flush := tester.NewContext(t)
defer flush()
status := support.CreateStatus(
ctx,
support.Restore, folder,
support.CollectionMetrics{
Objects: objects,
Successes: success,
Bytes: 0,
},
"statusTestTask")
ctrl.UpdateStatus(status)
}
func (suite *DisconnectedUnitSuite) TestController_Status() {
t := suite.T()
ctrl := Controller{wg: &sync.WaitGroup{}}
// Two tasks
ctrl.incrementAwaitingMessages()
ctrl.incrementAwaitingMessages()
// Each helper task processes 4 objects, 1 success, 3 errors, 1 folders
go statusTestTask(t, &ctrl, 4, 1, 1)
go statusTestTask(t, &ctrl, 4, 1, 1)
stats := ctrl.Wait()
assert.NotEmpty(t, ctrl.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, stats.Objects)
// Expect 2 success
assert.Equal(t, 2, stats.Successes)
// Expect 2 folders
assert.Equal(t, 2, stats.Folders)
}
func (suite *DisconnectedUnitSuite) TestVerifyBackupInputs_allServices() {
sites := []string{"abc.site.foo", "bar.site.baz"}
tests := []struct {
name string
excludes func(t *testing.T) selectors.Selector
filters func(t *testing.T) selectors.Selector
includes func(t *testing.T) selectors.Selector
checkError assert.ErrorAssertionFunc
}{
{
name: "Valid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
},
{
name: "Invalid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
},
{
name: "valid sites",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Include(sel.AllData())
return sel.Selector
},
},
{
name: "invalid sites",
checkError: assert.Error,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Include(sel.AllData())
return sel.Selector
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
err := verifyBackupInputs(test.excludes(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.filters(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.includes(t), sites)
test.checkError(t, err, clues.ToCore(err))
})
}
}

0
src/internal/m365/exchange/contact_container_cache.go → src/internal/m365/exchange/contacts_container_cache.go
View File

0
src/internal/m365/exchange/event_container_cache.go → src/internal/m365/exchange/events_container_cache.go
View File

181
src/internal/m365/graph_connector_disconnected_test.go
View File

@ -1,181 +0,0 @@
package m365
import (
"sync"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
)
// ---------------------------------------------------------------
// Disconnected Test Section
// ---------------------------------------------------------------
type DisconnectedSuite struct {
tester.Suite
}
func TestSuite(t *testing.T) {
s := &DisconnectedSuite{
Suite: tester.NewUnitSuite(t),
}
suite.Run(t, s)
}
func statusTestTask(
t *testing.T,
ctrl *Controller,
objects, success, folder int,
) {
ctx, flush := tester.NewContext(t)
defer flush()
status := support.CreateStatus(
ctx,
support.Restore, folder,
support.CollectionMetrics{
Objects: objects,
Successes: success,
Bytes: 0,
},
"statusTestTask")
ctrl.UpdateStatus(status)
}
func (suite *DisconnectedSuite) TestController_Status() {
t := suite.T()
ctrl := Controller{wg: &sync.WaitGroup{}}
// Two tasks
ctrl.incrementAwaitingMessages()
ctrl.incrementAwaitingMessages()
// Each helper task processes 4 objects, 1 success, 3 errors, 1 folders
go statusTestTask(t, &ctrl, 4, 1, 1)
go statusTestTask(t, &ctrl, 4, 1, 1)
stats := ctrl.Wait()
assert.NotEmpty(t, ctrl.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, stats.Objects)
// Expect 2 success
assert.Equal(t, 2, stats.Successes)
// Expect 2 folders
assert.Equal(t, 2, stats.Folders)
}
func (suite *DisconnectedSuite) TestVerifyBackupInputs_allServices() {
sites := []string{"abc.site.foo", "bar.site.baz"}
tests := []struct {
name string
excludes func(t *testing.T) selectors.Selector
filters func(t *testing.T) selectors.Selector
includes func(t *testing.T) selectors.Selector
checkError assert.ErrorAssertionFunc
}{
{
name: "Valid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
},
{
name: "Invalid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
},
{
name: "valid sites",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Include(sel.AllData())
return sel.Selector
},
},
{
name: "invalid sites",
checkError: assert.Error,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Include(sel.AllData())
return sel.Selector
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
err := verifyBackupInputs(test.excludes(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.filters(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.includes(t), sites)
test.checkError(t, err, clues.ToCore(err))
})
}
}

26
src/internal/m365/graph_connector_helper_test.go → src/internal/m365/helper_test.go
View File

@ -19,7 +19,9 @@ import (
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/onedrive"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -102,15 +104,15 @@ func testElementsMatch[T any](
type restoreBackupInfo struct { type restoreBackupInfo struct {
name string name string
service path.ServiceType service path.ServiceType
collections []ColInfo collections []m365Stub.ColInfo
resourceCat resource.Category resourceCat resource.Category
} }
type restoreBackupInfoMultiVersion struct { type restoreBackupInfoMultiVersion struct {
service path.ServiceType service path.ServiceType
collectionsLatest []ColInfo collectionsLatest []m365Stub.ColInfo
collectionsPrevious []ColInfo collectionsPrevious []m365Stub.ColInfo
resource resource.Category resourceCat resource.Category
backupVersion int backupVersion int
} }
@ -686,7 +688,7 @@ func compareDriveItem(
t *testing.T, t *testing.T,
expected map[string][]byte, expected map[string][]byte,
item data.Stream, item data.Stream,
config ConfigInfo, config m365Stub.ConfigInfo,
rootDir bool, rootDir bool,
) bool { ) bool {
// Skip Drive permissions in the folder that used to be the root. We don't // Skip Drive permissions in the folder that used to be the root. We don't
@ -793,7 +795,7 @@ func compareDriveItem(
return true return true
} }
var fileData testOneDriveData var fileData odStub.FileData
err = json.Unmarshal(buf, &fileData) err = json.Unmarshal(buf, &fileData)
if !assert.NoError(t, err, "unmarshalling file data for file", name, clues.ToCore(err)) { if !assert.NoError(t, err, "unmarshalling file data for file", name, clues.ToCore(err)) {
@ -829,7 +831,7 @@ func compareItem(
service path.ServiceType, service path.ServiceType,
category path.CategoryType, category path.CategoryType,
item data.Stream, item data.Stream,
config ConfigInfo, config m365Stub.ConfigInfo,
rootDir bool, rootDir bool,
) bool { ) bool {
if mt, ok := item.(data.StreamModTime); ok { if mt, ok := item.(data.StreamModTime); ok {
@ -923,7 +925,7 @@ func checkCollections(
expectedItems int, expectedItems int,
expected map[string]map[string][]byte, expected map[string]map[string][]byte,
got []data.BackupCollection, got []data.BackupCollection,
config ConfigInfo, config m365Stub.ConfigInfo,
) int { ) int {
collectionsWithItems := []data.BackupCollection{} collectionsWithItems := []data.BackupCollection{}
@ -985,7 +987,7 @@ func checkCollections(
checkHasCollections(t, expected, collectionsWithItems) checkHasCollections(t, expected, collectionsWithItems)
// Return how many metadata files were skipped so we can account for it in the // Return how many metadata files were skipped so we can account for it in the
// check on Controller status. // check on controller status.
return skipped return skipped
} }
@ -1152,11 +1154,11 @@ func getSelectorWith(
} }
} }
func loadController(ctx context.Context, t *testing.T, r resource.Category) *Controller { func newController(ctx context.Context, t *testing.T, r resource.Category) *Controller {
a := tester.NewM365Account(t) a := tester.NewM365Account(t)
connector, err := NewController(ctx, a, r) controller, err := NewController(ctx, a, r)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
return connector return controller
} }

24
src/internal/m365/mock/collection.go Normal file
View File

@ -0,0 +1,24 @@
package mock
import (
"context"
"github.com/alcionai/corso/src/internal/data"
)
type RestoreCollection struct {
data.Collection
AuxItems map[string]data.Stream
}
func (rc RestoreCollection) FetchItemByName(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.AuxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}

0
src/internal/m365/onedrive/drive.go → src/internal/m365/onedrive/item_collector.go
View File

7
src/internal/m365/onedrive/drive_test.go → src/internal/m365/onedrive/item_collector_test.go
View File

@ -28,13 +28,12 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api/mock" "github.com/alcionai/corso/src/pkg/services/m365/api/mock"
) )
// Unit tests type ItemCollectorUnitSuite struct {
type OneDriveUnitSuite struct {
tester.Suite tester.Suite
} }
func TestOneDriveUnitSuite(t *testing.T) { func TestOneDriveUnitSuite(t *testing.T) {
suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)}) suite.Run(t, &ItemCollectorUnitSuite{Suite: tester.NewUnitSuite(t)})
} }
const ( const (
@ -51,7 +50,7 @@ func odErr(code string) *odataerrors.ODataError {
return odErr return odErr
} }
func (suite *OneDriveUnitSuite) TestDrives() { func (suite *ItemCollectorUnitSuite) TestDrives() {
t := suite.T() t := suite.T()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)

174
src/internal/m365/graph_connector_onedrive_test_helper.go → src/internal/m365/onedrive/stub/stub.go
View File

@ -1,4 +1,4 @@
package m365 package stub
import ( import (
"encoding/json" "encoding/json"
@ -6,11 +6,10 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/google/uuid" "github.com/google/uuid"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/data"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
@ -61,59 +60,59 @@ type ItemData struct {
Perms PermData Perms PermData
} }
type OnedriveColInfo struct { type ColInfo struct {
PathElements []string PathElements []string
Perms PermData Perms PermData
Files []ItemData Files []ItemData
Folders []ItemData Folders []ItemData
} }
type onedriveCollection struct { type collection struct {
service path.ServiceType Service path.ServiceType
PathElements []string PathElements []string
items []ItemInfo Items []m365Stub.ItemInfo
aux []ItemInfo Aux []m365Stub.ItemInfo
backupVersion int BackupVersion int
} }
func (c onedriveCollection) collection() ColInfo { func (c collection) ColInfo() m365Stub.ColInfo {
cat := path.FilesCategory cat := path.FilesCategory
if c.service == path.SharePointService { if c.Service == path.SharePointService {
cat = path.LibrariesCategory cat = path.LibrariesCategory
} }
return ColInfo{ return m365Stub.ColInfo{
PathElements: c.PathElements, PathElements: c.PathElements,
Category: cat, Category: cat,
Items: c.items, Items: c.Items,
AuxItems: c.aux, AuxItems: c.Aux,
} }
} }
func NewOneDriveCollection( func NewCollection(
service path.ServiceType, service path.ServiceType,
PathElements []string, PathElements []string,
backupVersion int, backupVersion int,
) *onedriveCollection { ) *collection {
return &onedriveCollection{ return &collection{
service: service, Service: service,
PathElements: PathElements, PathElements: PathElements,
backupVersion: backupVersion, BackupVersion: backupVersion,
} }
} }
func DataForInfo( func DataForInfo(
service path.ServiceType, service path.ServiceType,
cols []OnedriveColInfo, cols []ColInfo,
backupVersion int, backupVersion int,
) ([]ColInfo, error) { ) ([]m365Stub.ColInfo, error) {
var ( var (
res []ColInfo res []m365Stub.ColInfo
err error err error
) )
for _, c := range cols { for _, c := range cols {
onedriveCol := NewOneDriveCollection(service, c.PathElements, backupVersion) onedriveCol := NewCollection(service, c.PathElements, backupVersion)
for _, f := range c.Files { for _, f := range c.Files {
_, err = onedriveCol.withFile(f.Name, f.Data, f.Perms) _, err = onedriveCol.withFile(f.Name, f.Data, f.Perms)
@ -134,18 +133,18 @@ func DataForInfo(
return res, err return res, err
} }
res = append(res, onedriveCol.collection()) res = append(res, onedriveCol.ColInfo())
} }
return res, nil return res, nil
} }
func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermData) (*onedriveCollection, error) { func (c *collection) withFile(name string, fileData []byte, perm PermData) (*collection, error) {
switch c.backupVersion { switch c.BackupVersion {
case 0: case 0:
// Lookups will occur using the most recent version of things so we need // Lookups will occur using the most recent version of things so we need
// the embedded file name to match that. // the embedded file name to match that.
item, err := onedriveItemWithData( item, err := FileWithData(
name, name,
name+metadata.DataFileSuffix, name+metadata.DataFileSuffix,
fileData) fileData)
@ -153,12 +152,12 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat
return c, err return c, err
} }
c.items = append(c.items, item) c.Items = append(c.Items, item)
// v1-5, early metadata design // v1-5, early metadata design
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker, case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker,
version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName: version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName:
items, err := onedriveItemWithData( items, err := FileWithData(
name+metadata.DataFileSuffix, name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix, name+metadata.DataFileSuffix,
fileData) fileData)
@ -166,24 +165,24 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat
return c, err return c, err
} }
c.items = append(c.items, items) c.Items = append(c.Items, items)
md, err := onedriveMetadata( md, err := ItemWithMetadata(
"", "",
name+metadata.MetaFileSuffix, name+metadata.MetaFileSuffix,
name+metadata.MetaFileSuffix, name+metadata.MetaFileSuffix,
perm, perm,
c.backupVersion >= versionPermissionSwitchedToID) c.BackupVersion >= versionPermissionSwitchedToID)
if err != nil { if err != nil {
return c, err return c, err
} }
c.items = append(c.items, md) c.Items = append(c.Items, md)
c.aux = append(c.aux, md) c.Aux = append(c.Aux, md)
// v6+ current metadata design // v6+ current metadata design
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID: case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
item, err := onedriveItemWithData( item, err := FileWithData(
name+metadata.DataFileSuffix, name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix, name+metadata.DataFileSuffix,
fileData) fileData)
@ -191,50 +190,50 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat
return c, err return c, err
} }
c.items = append(c.items, item) c.Items = append(c.Items, item)
md, err := onedriveMetadata( md, err := ItemWithMetadata(
name, name,
name+metadata.MetaFileSuffix, name+metadata.MetaFileSuffix,
name, name,
perm, perm,
c.backupVersion >= versionPermissionSwitchedToID) c.BackupVersion >= versionPermissionSwitchedToID)
if err != nil { if err != nil {
return c, err return c, err
} }
c.items = append(c.items, md) c.Items = append(c.Items, md)
c.aux = append(c.aux, md) c.Aux = append(c.Aux, md)
default: default:
return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.backupVersion)) return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.BackupVersion))
} }
return c, nil return c, nil
} }
func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCollection, error) { func (c *collection) withFolder(name string, perm PermData) (*collection, error) {
switch c.backupVersion { switch c.BackupVersion {
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName, case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID: version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
return c, nil return c, nil
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker: case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
item, err := onedriveMetadata( item, err := ItemWithMetadata(
"", "",
name+metadata.DirMetaFileSuffix, name+metadata.DirMetaFileSuffix,
name+metadata.DirMetaFileSuffix, name+metadata.DirMetaFileSuffix,
perm, perm,
c.backupVersion >= versionPermissionSwitchedToID) c.BackupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, item) c.Items = append(c.Items, item)
if err != nil { if err != nil {
return c, err return c, err
} }
default: default:
return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.backupVersion)) return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.BackupVersion))
} }
return c, nil return c, nil
@ -242,17 +241,17 @@ func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCo
// withPermissions adds permissions to the folder represented by this // withPermissions adds permissions to the folder represented by this
// onedriveCollection. // onedriveCollection.
func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection, error) { func (c *collection) withPermissions(perm PermData) (*collection, error) {
// These versions didn't store permissions for the folder or didn't store them // These versions didn't store permissions for the folder or didn't store them
// in the folder's collection. // in the folder's collection.
if c.backupVersion < version.OneDrive4DirIncludesPermissions { if c.BackupVersion < version.OneDrive4DirIncludesPermissions {
return c, nil return c, nil
} }
name := c.PathElements[len(c.PathElements)-1] name := c.PathElements[len(c.PathElements)-1]
metaName := name metaName := name
if c.backupVersion >= version.OneDrive5DirMetaNoName { if c.BackupVersion >= version.OneDrive5DirMetaNoName {
// We switched to just .dirmeta for metadata file names. // We switched to just .dirmeta for metadata file names.
metaName = "" metaName = ""
} }
@ -261,98 +260,63 @@ func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection
return c, nil return c, nil
} }
md, err := onedriveMetadata( md, err := ItemWithMetadata(
name, name,
metaName+metadata.DirMetaFileSuffix, metaName+metadata.DirMetaFileSuffix,
metaName+metadata.DirMetaFileSuffix, metaName+metadata.DirMetaFileSuffix,
perm, perm,
c.backupVersion >= versionPermissionSwitchedToID) c.BackupVersion >= versionPermissionSwitchedToID)
if err != nil { if err != nil {
return c, err return c, err
} }
c.items = append(c.items, md) c.Items = append(c.Items, md)
c.aux = append(c.aux, md) c.Aux = append(c.Aux, md)
return c, err return c, err
} }
type testOneDriveData struct { type FileData struct {
FileName string `json:"fileName,omitempty"` FileName string `json:"fileName,omitempty"`
Data []byte `json:"data,omitempty"` Data []byte `json:"data,omitempty"`
} }
func onedriveItemWithData( func FileWithData(
name, lookupKey string, name, lookupKey string,
fileData []byte, fileData []byte,
) (ItemInfo, error) { ) (m365Stub.ItemInfo, error) {
content := testOneDriveData{ content := FileData{
FileName: lookupKey, FileName: lookupKey,
Data: fileData, Data: fileData,
} }
serialized, err := json.Marshal(content) serialized, err := json.Marshal(content)
if err != nil { if err != nil {
return ItemInfo{}, clues.Stack(err) return m365Stub.ItemInfo{}, clues.Stack(err)
} }
return ItemInfo{ return m365Stub.ItemInfo{
name: name, Name: name,
data: serialized, Data: serialized,
lookupKey: lookupKey, LookupKey: lookupKey,
}, nil }, nil
} }
func onedriveMetadata( func ItemWithMetadata(
fileName, itemID, lookupKey string, fileName, itemID, lookupKey string,
perm PermData, perm PermData,
permUseID bool, permUseID bool,
) (ItemInfo, error) { ) (m365Stub.ItemInfo, error) {
testMeta := getMetadata(fileName, perm, permUseID) testMeta := getMetadata(fileName, perm, permUseID)
testMetaJSON, err := json.Marshal(testMeta) testMetaJSON, err := json.Marshal(testMeta)
if err != nil { if err != nil {
return ItemInfo{}, clues.Wrap(err, "marshalling metadata") return m365Stub.ItemInfo{}, clues.Wrap(err, "marshalling metadata")
} }
return ItemInfo{ return m365Stub.ItemInfo{
name: itemID, Name: itemID,
data: testMetaJSON, Data: testMetaJSON,
lookupKey: lookupKey, LookupKey: lookupKey,
}, nil }, nil
} }
func GetCollectionsAndExpected(
config ConfigInfo,
testCollections []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections []data.RestoreCollection
expectedData = map[string]map[string][]byte{}
totalItems = 0
totalKopiaItems = 0
)
for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData, err := collectionsForInfo(
config.Service,
config.Tenant,
owner,
config.RestoreCfg,
testCollections,
backupVersion,
)
if err != nil {
return totalItems, totalKopiaItems, collections, expectedData, err
}
collections = append(collections, ownerCollections...)
totalItems += numItems
totalKopiaItems += kopiaItems
maps.Copy(expectedData, userExpectedData)
}
return totalItems, totalKopiaItems, collections, expectedData, nil
}

179
src/internal/m365/graph_connector_onedrive_test.go → src/internal/m365/onedrive_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
@ -96,17 +97,17 @@ type oneDriveSuite interface {
} }
type suiteInfoImpl struct { type suiteInfoImpl struct {
ac api.Client ac api.Client
controller *Controller controller *Controller
resourceOwner string resourceOwner string
resourceCat resource.Category resourceCategory resource.Category
secondaryUser string secondaryUser string
secondaryUserID string secondaryUserID string
service path.ServiceType service path.ServiceType
tertiaryUser string tertiaryUser string
tertiaryUserID string tertiaryUserID string
user string user string
userID string userID string
} }
func NewSuiteInfoImpl( func NewSuiteInfoImpl(
@ -115,22 +116,22 @@ func NewSuiteInfoImpl(
resourceOwner string, resourceOwner string,
service path.ServiceType, service path.ServiceType,
) suiteInfoImpl { ) suiteInfoImpl {
rc := resource.Users rsc := resource.Users
if service == path.SharePointService { if service == path.SharePointService {
rc = resource.Sites rsc = resource.Sites
} }
ctrl := loadController(ctx, t, rc) ctrl := newController(ctx, t, rsc)
return suiteInfoImpl{ return suiteInfoImpl{
ac: ctrl.AC, ac: ctrl.AC,
controller: ctrl, controller: ctrl,
resourceOwner: resourceOwner, resourceOwner: resourceOwner,
resourceCat: rc, resourceCategory: rsc,
secondaryUser: tester.SecondaryM365UserID(t), secondaryUser: tester.SecondaryM365UserID(t),
service: service, service: service,
tertiaryUser: tester.TertiaryM365UserID(t), tertiaryUser: tester.TertiaryM365UserID(t),
user: tester.M365UserID(t), user: tester.M365UserID(t),
} }
} }
@ -163,7 +164,7 @@ func (si suiteInfoImpl) Service() path.ServiceType {
} }
func (si suiteInfoImpl) Resource() resource.Category { func (si suiteInfoImpl) Resource() resource.Category {
return si.resourceCat return si.resourceCategory
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -402,16 +403,16 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
folderBName, folderBName,
} }
cols := []OnedriveColInfo{ cols := []stub.ColInfo{
{ {
PathElements: rootPath, PathElements: rootPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileAData, Data: fileAData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderAName, Name: folderAName,
}, },
@ -422,13 +423,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
}, },
{ {
PathElements: folderAPath, PathElements: folderAPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileBData, Data: fileBData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderBName, Name: folderBName,
}, },
@ -436,13 +437,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
}, },
{ {
PathElements: subfolderBPath, PathElements: subfolderBPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileCData, Data: fileCData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderAName, Name: folderAName,
}, },
@ -450,7 +451,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
}, },
{ {
PathElements: subfolderAPath, PathElements: subfolderAPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileDData, Data: fileDData,
@ -459,7 +460,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
}, },
{ {
PathElements: folderBPath, PathElements: folderBPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileEData, Data: fileEData,
@ -468,18 +469,18 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
}, },
} }
expected, err := DataForInfo(suite.Service(), cols, version.Backup) expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() { suite.Run(fmt.Sprintf("Version%d", vn), func() {
t := suite.T() t := suite.T()
input, err := DataForInfo(suite.Service(), cols, vn) input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.Service(), service: suite.Service(),
resource: suite.Resource(), resourceCat: suite.Resource(),
backupVersion: vn, backupVersion: vn,
collectionsPrevious: input, collectionsPrevious: input,
collectionsLatest: expected, collectionsLatest: expected,
@ -549,15 +550,15 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
folderCName, folderCName,
} }
cols := []OnedriveColInfo{ cols := []stub.ColInfo{
{ {
PathElements: rootPath, PathElements: rootPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
// Test restoring a file that doesn't inherit permissions. // Test restoring a file that doesn't inherit permissions.
Name: fileName, Name: fileName,
Data: fileAData, Data: fileAData,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
@ -570,13 +571,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
Data: fileBData, Data: fileBData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderBName, Name: folderBName,
}, },
{ {
Name: folderAName, Name: folderAName,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -584,7 +585,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
}, },
{ {
Name: folderCName, Name: folderCName,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -594,23 +595,23 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
}, },
{ {
PathElements: folderBPath, PathElements: folderBPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
// Test restoring a file in a non-root folder that doesn't inherit // Test restoring a file in a non-root folder that doesn't inherit
// permissions. // permissions.
Name: fileName, Name: fileName,
Data: fileBData, Data: fileBData,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderAName, Name: folderAName,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -624,18 +625,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// // Tests a folder that has permissions with an item in the folder with // // Tests a folder that has permissions with an item in the folder with
// // the same permissions. // // the same permissions.
// pathElements: subfolderAPath, // pathElements: subfolderAPath,
// files: []itemData{ // files: []stub.ItemData{
// { // {
// name: fileName, // name: fileName,
// data: fileDData, // data: fileDData,
// perms: permData{ // perms: stub.PermData{
// user: secondaryUserName, // user: secondaryUserName,
// entityID: secondaryUserID, // entityID: secondaryUserID,
// roles: readPerm, // roles: readPerm,
// }, // },
// }, // },
// }, // },
// Perms: PermData{ // Perms: stub.PermData{
// User: secondaryUserName, // User: secondaryUserName,
// EntityID: secondaryUserID, // EntityID: secondaryUserID,
// Roles: readPerm, // Roles: readPerm,
@ -645,18 +646,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Tests a folder that has permissions with an item in the folder with // Tests a folder that has permissions with an item in the folder with
// the different permissions. // the different permissions.
PathElements: folderAPath, PathElements: folderAPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileEData, Data: fileEData,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
}, },
}, },
}, },
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -666,13 +667,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Tests a folder that has permissions with an item in the folder with // Tests a folder that has permissions with an item in the folder with
// no permissions. // no permissions.
PathElements: folderCPath, PathElements: folderCPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileAData, Data: fileAData,
}, },
}, },
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: readPerm, Roles: readPerm,
@ -680,7 +681,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
}, },
} }
expected, err := DataForInfo(suite.Service(), cols, version.Backup) expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
bss := suite.Service().String() bss := suite.Service().String()
@ -690,12 +691,12 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Ideally this can always be true or false and still // Ideally this can always be true or false and still
// work, but limiting older versions to use emails so as // work, but limiting older versions to use emails so as
// to validate that flow as well. // to validate that flow as well.
input, err := DataForInfo(suite.Service(), cols, vn) input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.Service(), service: suite.Service(),
resource: suite.Resource(), resourceCat: suite.Resource(),
backupVersion: vn, backupVersion: vn,
collectionsPrevious: input, collectionsPrevious: input,
collectionsLatest: expected, collectionsLatest: expected,
@ -730,18 +731,18 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
suite.Service(), suite.Service(),
suite.ResourceOwner()) suite.ResourceOwner())
inputCols := []OnedriveColInfo{ inputCols := []stub.ColInfo{
{ {
PathElements: []string{ PathElements: []string{
odConsts.DrivesPathDir, odConsts.DrivesPathDir,
driveID, driveID,
odConsts.RootPathDir, odConsts.RootPathDir,
}, },
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileAData, Data: fileAData,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
@ -751,14 +752,14 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
}, },
} }
expectedCols := []OnedriveColInfo{ expectedCols := []stub.ColInfo{
{ {
PathElements: []string{ PathElements: []string{
odConsts.DrivesPathDir, odConsts.DrivesPathDir,
driveID, driveID,
odConsts.RootPathDir, odConsts.RootPathDir,
}, },
Files: []ItemData{ Files: []stub.ItemData{
{ {
// No permissions on the output since they weren't restored. // No permissions on the output since they weren't restored.
Name: fileName, Name: fileName,
@ -768,19 +769,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
}, },
} }
expected, err := DataForInfo(suite.Service(), expectedCols, version.Backup) expected, err := stub.DataForInfo(suite.Service(), expectedCols, version.Backup)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
bss := suite.Service().String() bss := suite.Service().String()
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() { suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() {
t := suite.T() t := suite.T()
input, err := DataForInfo(suite.Service(), inputCols, vn) input, err := stub.DataForInfo(suite.Service(), inputCols, vn)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.Service(), service: suite.Service(),
resource: suite.Resource(), resourceCat: suite.Resource(),
backupVersion: vn, backupVersion: vn,
collectionsPrevious: input, collectionsPrevious: input,
collectionsLatest: expected, collectionsLatest: expected,
@ -855,11 +856,11 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
folderCName, folderCName,
} }
fileSet := []ItemData{ fileSet := []stub.ItemData{
{ {
Name: "file-custom", Name: "file-custom",
Data: fileAData, Data: fileAData,
Perms: PermData{ Perms: stub.PermData{
User: secondaryUserName, User: secondaryUserName,
EntityID: secondaryUserID, EntityID: secondaryUserID,
Roles: writePerm, Roles: writePerm,
@ -869,14 +870,14 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
{ {
Name: "file-inherited", Name: "file-inherited",
Data: fileAData, Data: fileAData,
Perms: PermData{ Perms: stub.PermData{
SharingMode: metadata.SharingModeInherited, SharingMode: metadata.SharingModeInherited,
}, },
}, },
{ {
Name: "file-empty", Name: "file-empty",
Data: fileAData, Data: fileAData,
Perms: PermData{ Perms: stub.PermData{
SharingMode: metadata.SharingModeCustom, SharingMode: metadata.SharingModeCustom,
}, },
}, },
@ -900,23 +901,23 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
// - inherted-permission-file // - inherted-permission-file
// - empty-permission-file (empty/empty might have interesting behavior) // - empty-permission-file (empty/empty might have interesting behavior)
cols := []OnedriveColInfo{ cols := []stub.ColInfo{
{ {
PathElements: rootPath, PathElements: rootPath,
Files: []ItemData{}, Files: []stub.ItemData{},
Folders: []ItemData{ Folders: []stub.ItemData{
{Name: folderAName}, {Name: folderAName},
}, },
}, },
{ {
PathElements: folderAPath, PathElements: folderAPath,
Files: fileSet, Files: fileSet,
Folders: []ItemData{ Folders: []stub.ItemData{
{Name: folderAName}, {Name: folderAName},
{Name: folderBName}, {Name: folderBName},
{Name: folderCName}, {Name: folderCName},
}, },
Perms: PermData{ Perms: stub.PermData{
User: tertiaryUserName, User: tertiaryUserName,
EntityID: tertiaryUserID, EntityID: tertiaryUserID,
Roles: readPerm, Roles: readPerm,
@ -925,7 +926,7 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
{ {
PathElements: subfolderAAPath, PathElements: subfolderAAPath,
Files: fileSet, Files: fileSet,
Perms: PermData{ Perms: stub.PermData{
User: tertiaryUserName, User: tertiaryUserName,
EntityID: tertiaryUserID, EntityID: tertiaryUserID,
Roles: writePerm, Roles: writePerm,
@ -935,20 +936,20 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
{ {
PathElements: subfolderABPath, PathElements: subfolderABPath,
Files: fileSet, Files: fileSet,
Perms: PermData{ Perms: stub.PermData{
SharingMode: metadata.SharingModeInherited, SharingMode: metadata.SharingModeInherited,
}, },
}, },
{ {
PathElements: subfolderACPath, PathElements: subfolderACPath,
Files: fileSet, Files: fileSet,
Perms: PermData{ Perms: stub.PermData{
SharingMode: metadata.SharingModeCustom, SharingMode: metadata.SharingModeCustom,
}, },
}, },
} }
expected, err := DataForInfo(suite.Service(), cols, version.Backup) expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
bss := suite.Service().String() bss := suite.Service().String()
@ -958,12 +959,12 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
// Ideally this can always be true or false and still // Ideally this can always be true or false and still
// work, but limiting older versions to use emails so as // work, but limiting older versions to use emails so as
// to validate that flow as well. // to validate that flow as well.
input, err := DataForInfo(suite.Service(), cols, vn) input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.Service(), service: suite.Service(),
resource: suite.Resource(), resourceCat: suite.Resource(),
backupVersion: vn, backupVersion: vn,
collectionsPrevious: input, collectionsPrevious: input,
collectionsLatest: expected, collectionsLatest: expected,
@ -1018,16 +1019,16 @@ func testRestoreFolderNamedFolderRegression(
folderBName, folderBName,
} }
cols := []OnedriveColInfo{ cols := []stub.ColInfo{
{ {
PathElements: rootPath, PathElements: rootPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileAData, Data: fileAData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderNamedFolder, Name: folderNamedFolder,
}, },
@ -1038,13 +1039,13 @@ func testRestoreFolderNamedFolderRegression(
}, },
{ {
PathElements: folderFolderPath, PathElements: folderFolderPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileBData, Data: fileBData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderBName, Name: folderBName,
}, },
@ -1052,13 +1053,13 @@ func testRestoreFolderNamedFolderRegression(
}, },
{ {
PathElements: subfolderPath, PathElements: subfolderPath,
Files: []ItemData{ Files: []stub.ItemData{
{ {
Name: fileName, Name: fileName,
Data: fileCData, Data: fileCData,
}, },
}, },
Folders: []ItemData{ Folders: []stub.ItemData{
{ {
Name: folderNamedFolder, Name: folderNamedFolder,
}, },
@ -1066,19 +1067,19 @@ func testRestoreFolderNamedFolderRegression(
}, },
} }
expected, err := DataForInfo(suite.Service(), cols, version.Backup) expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
bss := suite.Service().String() bss := suite.Service().String()
for vn := startVersion; vn <= version.Backup; vn++ { for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() { suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() {
t := suite.T() t := suite.T()
input, err := DataForInfo(suite.Service(), cols, vn) input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{ testData := restoreBackupInfoMultiVersion{
service: suite.Service(), service: suite.Service(),
resource: suite.Resource(), resourceCat: suite.Resource(),
backupVersion: vn, backupVersion: vn,
collectionsPrevious: input, collectionsPrevious: input,
collectionsLatest: expected, collectionsLatest: expected,

2
src/internal/m365/sharepoint/api/pages.go
View File

@ -172,7 +172,7 @@ func RestoreSitePage(
itemData data.Stream, itemData data.Stream,
siteID, destName string, siteID, destName string,
) (details.ItemInfo, error) { ) (details.ItemInfo, error) {
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID())) ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID()))
defer end() defer end()
var ( var (

4
src/internal/m365/sharepoint/collection.go
View File

@ -257,7 +257,7 @@ func (sc *Collection) retrieveLists(
sc.data <- &Item{ sc.data <- &Item{
id: ptr.Val(lst.GetId()), id: ptr.Val(lst.GetId()),
data: io.NopCloser(bytes.NewReader(byteArray)), data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointListInfo(lst, size), info: listToSPInfo(lst, size),
modTime: t, modTime: t,
} }
@ -320,7 +320,7 @@ func (sc *Collection) retrievePages(
sc.data <- &Item{ sc.data <- &Item{
id: ptr.Val(pg.GetId()), id: ptr.Val(pg.GetId()),
data: io.NopCloser(bytes.NewReader(byteArray)), data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointPageInfo(pg, root, size), info: pageToSPInfo(pg, root, size),
modTime: ptr.OrNow(pg.GetLastModifiedDateTime()), modTime: ptr.OrNow(pg.GetLastModifiedDateTime()),
} }

4
src/internal/m365/sharepoint/collection_test.go
View File

@ -116,7 +116,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
data := &Item{ data := &Item{
id: name, id: name,
data: io.NopCloser(bytes.NewReader(byteArray)), data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointListInfo(listing, int64(len(byteArray))), info: listToSPInfo(listing, int64(len(byteArray))),
} }
return data return data
@ -205,7 +205,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
listData := &Item{ listData := &Item{
id: testName, id: testName,
data: io.NopCloser(bytes.NewReader(byteArray)), data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointListInfo(listing, int64(len(byteArray))), info: listToSPInfo(listing, int64(len(byteArray))),
} }
destName := tester.DefaultTestRestoreConfig("").Location destName := tester.DefaultTestRestoreConfig("").Location

28
src/internal/m365/sharepoint/list_info.go
View File

@ -1,28 +0,0 @@
package sharepoint
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/backup/details"
)
// sharePointListInfo translates models.Listable metadata into searchable content
// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
func sharePointListInfo(lst models.Listable, size int64) *details.SharePointInfo {
var (
name = ptr.Val(lst.GetDisplayName())
webURL = ptr.Val(lst.GetWebUrl())
created = ptr.Val(lst.GetCreatedDateTime())
modified = ptr.Val(lst.GetLastModifiedDateTime())
)
return &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: name,
Created: created,
Modified: modified,
WebURL: webURL,
Size: size,
}
}

59
src/internal/m365/sharepoint/list_info_test.go
View File

@ -1,59 +0,0 @@
package sharepoint
import (
"testing"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
)
type SharePointInfoSuite struct {
tester.Suite
}
func TestSharePointInfoSuite(t *testing.T) {
suite.Run(t, &SharePointInfoSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *SharePointInfoSuite) TestSharePointInfo() {
tests := []struct {
name string
listAndDeets func() (models.Listable, *details.SharePointInfo)
}{
{
name: "Empty List",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
i := &details.SharePointInfo{ItemType: details.SharePointList}
return models.NewList(), i
},
}, {
name: "Only Name",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
aTitle := "Whole List"
listing := models.NewList()
listing.SetDisplayName(&aTitle)
i := &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: aTitle,
}
return listing, i
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
list, expected := test.listAndDeets()
info := sharePointListInfo(list, 10)
assert.Equal(t, expected.ItemType, info.ItemType)
assert.Equal(t, expected.ItemName, info.ItemName)
assert.Equal(t, expected.WebURL, info.WebURL)
})
}
}

21
src/internal/m365/sharepoint/list.go → src/internal/m365/sharepoint/lists.go
View File

@ -10,9 +10,30 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
) )
// listToSPInfo translates models.Listable metadata into searchable content
// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
func listToSPInfo(lst models.Listable, size int64) *details.SharePointInfo {
var (
name = ptr.Val(lst.GetDisplayName())
webURL = ptr.Val(lst.GetWebUrl())
created = ptr.Val(lst.GetCreatedDateTime())
modified = ptr.Val(lst.GetLastModifiedDateTime())
)
return &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: name,
Created: created,
Modified: modified,
WebURL: webURL,
Size: size,
}
}
type listTuple struct { type listTuple struct {
name string name string
id string id string

51
src/internal/m365/sharepoint/list_test.go → src/internal/m365/sharepoint/lists_test.go
View File

@ -4,21 +4,23 @@ import (
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
) )
type SharePointSuite struct { type ListsUnitSuite struct {
tester.Suite tester.Suite
creds account.M365Config creds account.M365Config
} }
func (suite *SharePointSuite) SetupSuite() { func (suite *ListsUnitSuite) SetupSuite() {
t := suite.T() t := suite.T()
a := tester.NewM365Account(t) a := tester.NewM365Account(t)
m365, err := a.M365Config() m365, err := a.M365Config()
@ -27,8 +29,8 @@ func (suite *SharePointSuite) SetupSuite() {
suite.creds = m365 suite.creds = m365
} }
func TestSharePointSuite(t *testing.T) { func TestListsUnitSuite(t *testing.T) {
suite.Run(t, &SharePointSuite{ suite.Run(t, &ListsUnitSuite{
Suite: tester.NewIntegrationSuite( Suite: tester.NewIntegrationSuite(
t, t,
[][]string{tester.M365AcctCredEnvs}, [][]string{tester.M365AcctCredEnvs},
@ -47,7 +49,7 @@ func TestSharePointSuite(t *testing.T) {
// to verify if these 2 calls are valid // to verify if these 2 calls are valid
// - fetchContentBaseTypes // - fetchContentBaseTypes
// - fetchColumnPositions // - fetchColumnPositions
func (suite *SharePointSuite) TestLoadList() { func (suite *ListsUnitSuite) TestLoadList() {
t := suite.T() t := suite.T()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
@ -63,3 +65,42 @@ func (suite *SharePointSuite) TestLoadList() {
assert.Greater(t, len(lists), 0) assert.Greater(t, len(lists), 0)
t.Logf("Length: %d\n", len(lists)) t.Logf("Length: %d\n", len(lists))
} }
func (suite *ListsUnitSuite) TestSharePointInfo() {
tests := []struct {
name string
listAndDeets func() (models.Listable, *details.SharePointInfo)
}{
{
name: "Empty List",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
i := &details.SharePointInfo{ItemType: details.SharePointList}
return models.NewList(), i
},
}, {
name: "Only Name",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
aTitle := "Whole List"
listing := models.NewList()
listing.SetDisplayName(&aTitle)
i := &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: aTitle,
}
return listing, i
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
list, expected := test.listAndDeets()
info := listToSPInfo(list, 10)
assert.Equal(t, expected.ItemType, info.ItemType)
assert.Equal(t, expected.ItemName, info.ItemName)
assert.Equal(t, expected.WebURL, info.WebURL)
})
}
}

4
src/internal/m365/sharepoint/pageInfo.go → src/internal/m365/sharepoint/pages.go
View File

@ -8,10 +8,10 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
) )
// sharePointPageInfo propagates metadata from the SharePoint Page data type // pageToSPInfo propagates metadata from the SharePoint Page data type
// into searchable content. // into searchable content.
// Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta // Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
func sharePointPageInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo { func pageToSPInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo {
var ( var (
name, prefix, webURL string name, prefix, webURL string
created, modified time.Time created, modified time.Time

16
src/internal/m365/sharepoint/pageInfo_test.go → src/internal/m365/sharepoint/pages_test.go
View File

@ -1,13 +1,25 @@
package sharepoint package sharepoint
import ( import (
"testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/m365/graph/betasdk/models" "github.com/alcionai/corso/src/internal/m365/graph/betasdk/models"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
) )
func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { type PagesUnitSuite struct {
tester.Suite
}
func TestPagesUnitSuite(t *testing.T) {
suite.Run(t, &PagesUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *PagesUnitSuite) TestSharePointInfo_Pages() {
tests := []struct { tests := []struct {
name string name string
pageAndDeets func() (models.SitePageable, *details.SharePointInfo) pageAndDeets func() (models.SitePageable, *details.SharePointInfo)
@ -39,7 +51,7 @@ func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() {
t := suite.T() t := suite.T()
paged, expected := test.pageAndDeets() paged, expected := test.pageAndDeets()
info := sharePointPageInfo(paged, "", 0) info := pageToSPInfo(paged, "", 0)
assert.Equal(t, expected.ItemType, info.ItemType) assert.Equal(t, expected.ItemType, info.ItemType)
assert.Equal(t, expected.ItemName, info.ItemName) assert.Equal(t, expected.ItemName, info.ItemName)
assert.Equal(t, expected.WebURL, info.WebURL) assert.Equal(t, expected.WebURL, info.WebURL)

12
src/internal/m365/sharepoint/restore.go
View File

@ -129,7 +129,7 @@ func restoreListItem(
itemData data.Stream, itemData data.Stream,
siteID, destName string, siteID, destName string,
) (details.ItemInfo, error) { ) (details.ItemInfo, error) {
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID())) ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID()))
defer end() defer end()
ctx = clues.Add(ctx, "list_item_id", itemData.UUID()) ctx = clues.Add(ctx, "list_item_id", itemData.UUID())
@ -190,7 +190,7 @@ func restoreListItem(
} }
} }
dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray))) dii.SharePoint = listToSPInfo(restoredList, int64(len(byteArray)))
return dii, nil return dii, nil
} }
@ -203,7 +203,7 @@ func RestoreListCollection(
deets *details.Builder, deets *details.Builder,
errs *fault.Bus, errs *fault.Bus,
) (support.CollectionMetrics, error) { ) (support.CollectionMetrics, error) {
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath())) ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath()))
defer end() defer end()
var ( var (
@ -214,7 +214,7 @@ func RestoreListCollection(
el = errs.Local() el = errs.Local()
) )
trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String()) trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String())
for { for {
if el.Failure() != nil { if el.Failure() != nil {
@ -285,8 +285,8 @@ func RestorePageCollection(
siteID = directory.ResourceOwner() siteID = directory.ResourceOwner()
) )
trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String()) trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String())
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath())) ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath()))
defer end() defer end()

225
src/internal/m365/graph_connector_test_helper.go → src/internal/m365/stub/stub.go
View File

@ -1,12 +1,14 @@
package m365 package stub
import ( import (
"bytes" "bytes"
"context"
"io" "io"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
"github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
@ -33,9 +35,9 @@ type ItemInfo struct {
// be the same before and after restoring the item in M365 and may not be // be the same before and after restoring the item in M365 and may not be
// the M365 ID. When restoring items out of place, the item is assigned a // the M365 ID. When restoring items out of place, the item is assigned a
// new ID making it unsuitable for a lookup key. // new ID making it unsuitable for a lookup key.
lookupKey string LookupKey string
name string Name string
data []byte Data []byte
} }
type ConfigInfo struct { type ConfigInfo struct {
@ -47,19 +49,113 @@ type ConfigInfo struct {
RestoreCfg control.RestoreConfig RestoreCfg control.RestoreConfig
} }
func mustToDataLayerPath( func GetCollectionsAndExpected(
service path.ServiceType, config ConfigInfo,
tenant, resourceOwner string, testCollections []ColInfo,
category path.CategoryType, backupVersion int,
elements []string, ) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
isItem bool, var (
) (path.Path, error) { collections []data.RestoreCollection
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...) expectedData = map[string]map[string][]byte{}
if err != nil { totalItems = 0
return nil, err totalKopiaItems = 0
)
for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo(
config.Service,
config.Tenant,
owner,
config.RestoreCfg,
testCollections,
backupVersion,
)
if err != nil {
return totalItems, totalKopiaItems, collections, expectedData, err
}
collections = append(collections, ownerCollections...)
totalItems += numItems
totalKopiaItems += kopiaItems
maps.Copy(expectedData, userExpectedData)
} }
return res, err return totalItems, totalKopiaItems, collections, expectedData, nil
}
func CollectionsForInfo(
service path.ServiceType,
tenant, user string,
restoreCfg control.RestoreConfig,
allInfo []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections = make([]data.RestoreCollection, 0, len(allInfo))
expectedData = make(map[string]map[string][]byte, len(allInfo))
totalItems = 0
kopiaEntries = 0
)
for _, info := range allInfo {
pth, err := path.Build(
tenant,
user,
service,
info.Category,
false,
info.PathElements...)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
mc := exchMock.NewCollection(pth, pth, len(info.Items))
baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
baseExpected := expectedData[baseDestPath.String()]
if baseExpected == nil {
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items))
baseExpected = expectedData[baseDestPath.String()]
}
for i := 0; i < len(info.Items); i++ {
mc.Names[i] = info.Items[i].Name
mc.Data[i] = info.Items[i].Data
baseExpected[info.Items[i].LookupKey] = info.Items[i].Data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].Name) {
continue
}
totalItems++
}
c := mock.RestoreCollection{
Collection: mc,
AuxItems: map[string]data.Stream{},
}
for _, aux := range info.AuxItems {
c.AuxItems[aux.Name] = &exchMock.Data{
ID: aux.Name,
Reader: io.NopCloser(bytes.NewReader(aux.Data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.Items)
}
return totalItems, kopiaEntries, collections, expectedData, nil
} }
// backupOutputPathFromRestore returns a path.Path denoting the location in // backupOutputPathFromRestore returns a path.Path denoting the location in
@ -86,102 +182,11 @@ func backupOutputPathFromRestore(
base = append(base, inputPath.Folders()...) base = append(base, inputPath.Folders()...)
} }
return mustToDataLayerPath( return path.Build(
inputPath.Service(),
inputPath.Tenant(), inputPath.Tenant(),
inputPath.ResourceOwner(), inputPath.ResourceOwner(),
inputPath.Service(),
inputPath.Category(), inputPath.Category(),
base,
false, false,
) base...)
}
// TODO(ashmrtn): Make this an actual mock class that can be used in other
// packages.
type mockRestoreCollection struct {
data.Collection
auxItems map[string]data.Stream
}
func (rc mockRestoreCollection) FetchItemByName(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.auxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}
func collectionsForInfo(
service path.ServiceType,
tenant, user string,
restoreCfg control.RestoreConfig,
allInfo []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections = make([]data.RestoreCollection, 0, len(allInfo))
expectedData = make(map[string]map[string][]byte, len(allInfo))
totalItems = 0
kopiaEntries = 0
)
for _, info := range allInfo {
pth, err := mustToDataLayerPath(
service,
tenant,
user,
info.Category,
info.PathElements,
false)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
mc := exchMock.NewCollection(pth, pth, len(info.Items))
baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
baseExpected := expectedData[baseDestPath.String()]
if baseExpected == nil {
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items))
baseExpected = expectedData[baseDestPath.String()]
}
for i := 0; i < len(info.Items); i++ {
mc.Names[i] = info.Items[i].name
mc.Data[i] = info.Items[i].data
baseExpected[info.Items[i].lookupKey] = info.Items[i].data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].name) {
continue
}
totalItems++
}
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.AuxItems {
c.auxItems[aux.name] = &exchMock.Data{
ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.Items)
}
return totalItems, kopiaEntries, collections, expectedData, nil
} }