larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

final cleanup for renaming/movement (#3612)

Browse Source 
This last step in the cleanup has two goals: first to minimize the number of arbitrarily named and located files so that code is better condensed and co-located.  This is all just file renaming and a minor amount of code copy-pasting.

The second is to create a centralized package to own the ColInfo type structs that we use to both stub out test data, and also generate factory data for cmds.  The current ownership is haphazard, and while this movement is a little more condensed, it's still jumping through some weird hoops to get things to work.  Treat it as one good step forward, and we'll have to return to polish it another time.  At least it'll be separated from the m365 folder at large this way, and more easily identified as supporting design rather than production usage.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* closes #1996

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-06-14 10:50:17 -06:00 committed by GitHub
parent 466698f096
commit bd0f6f9769
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 705 additions and 737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/cmd/factory/impl/common.go
View File

@ -18,7 +18,9 @@ import (
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365"
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/account"
@ -84,14 +86,14 @@ func generateAndRestoreItems(
items: items,
}}
dest := control.DefaultRestoreConfig(dttm.SafeForTesting)
dest.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.Location)
restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
restoreCfg.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
dataColls, err := buildCollections(
service,
tenantID, userID,
dest,
restoreCfg,
collections)
if err != nil {
return nil, err
@ -99,7 +101,7 @@ func generateAndRestoreItems(
print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, dataColls, errs)
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs)
}
// ------------------------------------------------------------------------------------------
@ -108,7 +110,7 @@ func generateAndRestoreItems(
func getControllerAndVerifyResourceOwner(
ctx context.Context,
rc resource.Category,
resourceCat resource.Category,
resourceOwner string,
) (
*m365.Controller,
@ -133,7 +135,7 @@ func getControllerAndVerifyResourceOwner(
return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details")
}
ctrl, err := m365.NewController(ctx, acct, rc)
ctrl, err := m365.NewController(ctx, acct, resourceCat)
if err != nil {
return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
}
@ -164,7 +166,7 @@ type collection struct {
func buildCollections(
service path.ServiceType,
tenant, user string,
dest control.RestoreConfig,
restoreCfg control.RestoreConfig,
colls []collection,
) ([]data.RestoreCollection, error) {
collections := make([]data.RestoreCollection, 0, len(colls))
@ -225,9 +227,9 @@ func generateAndRestoreDriveItems(
ctx, flush := tester.NewContext(nil)
defer flush()
dest := control.DefaultRestoreConfig(dttm.SafeForTesting)
dest.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.Location)
restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
restoreCfg.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
var driveID string
@ -249,7 +251,7 @@ func generateAndRestoreDriveItems(
}
var (
cols []m365.OnedriveColInfo
cols []odStub.ColInfo
rootPath = []string{"drives", driveID, "root:"}
folderAPath = []string{"drives", driveID, "root:", folderAName}
@ -263,15 +265,15 @@ func generateAndRestoreDriveItems(
)
for i := 0; i < count; i++ {
col := []m365.OnedriveColInfo{
col := []odStub.ColInfo{
// basic folder and file creation
{
PathElements: rootPath,
Files: []m365.ItemData{
Files: []odStub.ItemData{
{
Name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
Data: fileAData,
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
@ -282,13 +284,13 @@ func generateAndRestoreDriveItems(
Data: fileBData,
},
},
Folders: []m365.ItemData{
Folders: []odStub.ItemData{
{
Name: folderBName,
},
{
Name: folderAName,
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -296,7 +298,7 @@ func generateAndRestoreDriveItems(
},
{
Name: folderCName,
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -308,18 +310,18 @@ func generateAndRestoreDriveItems(
// a folder that has permissions with an item in the folder with
// the different permissions.
PathElements: folderAPath,
Files: []m365.ItemData{
Files: []odStub.ItemData{
{
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
Data: fileEData,
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
},
},
},
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -329,13 +331,13 @@ func generateAndRestoreDriveItems(
// a folder that has permissions with an item in the folder with
// no permissions.
PathElements: folderCPath,
Files: []m365.ItemData{
Files: []odStub.ItemData{
{
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
Data: fileAData,
},
},
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -343,23 +345,23 @@ func generateAndRestoreDriveItems(
},
{
PathElements: folderBPath,
Files: []m365.ItemData{
Files: []odStub.ItemData{
{
// restoring a file in a non-root folder that doesn't inherit
// permissions.
Name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
Data: fileBData,
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
},
},
},
Folders: []m365.ItemData{
Folders: []odStub.ItemData{
{
Name: folderAName,
Perms: m365.PermData{
Perms: odStub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -372,7 +374,7 @@ func generateAndRestoreDriveItems(
cols = append(cols, col...)
}
input, err := m365.DataForInfo(service, cols, version.Backup)
input, err := odStub.DataForInfo(service, cols, version.Backup)
if err != nil {
return nil, err
}
@ -389,7 +391,7 @@ func generateAndRestoreDriveItems(
ToggleFeatures: control.Toggles{},
}
config := m365.ConfigInfo{
config := m365Stub.ConfigInfo{
Opts: opts,
Resource: resource.Users,
Service: service,
@ -398,7 +400,7 @@ func generateAndRestoreDriveItems(
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
_, _, collections, _, err := m365.GetCollectionsAndExpected(
_, _, collections, _, err := m365Stub.GetCollectionsAndExpected(
config,
input,
version.Backup)
@ -406,5 +408,5 @@ func generateAndRestoreDriveItems(
return nil, err
}
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, dest, opts, collections, errs)
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs)
}

12
src/internal/m365/backup_test.go
View File

@ -66,7 +66,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() {
selUsers := []string{suite.user}
ctrl := loadController(ctx, suite.T(), resource.Users)
ctrl := newController(ctx, suite.T(), resource.Users)
tests := []struct {
name string
getSelector func(t *testing.T) selectors.Selector
@ -167,7 +167,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner()
defer flush()
owners := []string{"snuffleupagus"}
ctrl := loadController(ctx, suite.T(), resource.Users)
ctrl := newController(ctx, suite.T(), resource.Users)
tests := []struct {
name string
getSelector func(t *testing.T) selectors.Selector
@ -253,7 +253,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
defer flush()
selSites := []string{suite.site}
ctrl := loadController(ctx, suite.T(), resource.Sites)
ctrl := newController(ctx, suite.T(), resource.Sites)
tests := []struct {
name string
expected int
@ -348,7 +348,7 @@ func (suite *SPCollectionIntgSuite) SetupSuite() {
ctx, flush := tester.NewContext(suite.T())
defer flush()
suite.connector = loadController(ctx, suite.T(), resource.Sites)
suite.connector = newController(ctx, suite.T(), resource.Sites)
suite.user = tester.M365UserID(suite.T())
tester.LogTimeOfTest(suite.T())
@ -362,7 +362,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
var (
siteID = tester.M365SiteID(t)
ctrl = loadController(ctx, t, resource.Sites)
ctrl = newController(ctx, t, resource.Sites)
siteIDs = []string{siteID}
)
@ -409,7 +409,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
var (
siteID = tester.M365SiteID(t)
ctrl = loadController(ctx, t, resource.Sites)
ctrl = newController(ctx, t, resource.Sites)
siteIDs = []string{siteID}
)

349
src/internal/m365/controller_test.go
View File

@ -17,6 +17,7 @@ import (
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
"github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
@ -24,6 +25,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
)
// ---------------------------------------------------------------------------
@ -280,7 +282,7 @@ func (suite *ControllerIntegrationSuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.ctrl = loadController(ctx, t, resource.Users)
suite.ctrl = newController(ctx, t, resource.Users)
suite.user = tester.M365UserID(t)
suite.secondaryUser = tester.SecondaryM365UserID(t)
@ -407,7 +409,7 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() {
func runRestore(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
config ConfigInfo,
config stub.ConfigInfo,
backupVersion int,
collections []data.RestoreCollection,
numRestoreItems int,
@ -419,7 +421,7 @@ func runRestore(
start := time.Now()
restoreCtrl := loadController(ctx, t, config.Resource)
restoreCtrl := newController(ctx, t, config.Resource)
restoreSel := getSelectorWith(t, config.Service, config.ResourceOwners, true)
deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx,
@ -450,11 +452,11 @@ func runRestore(
func runBackupAndCompare(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
config ConfigInfo,
config stub.ConfigInfo,
expectedData map[string]map[string][]byte,
totalItems int,
totalKopiaItems int,
inputCollections []ColInfo,
inputCollections []stub.ColInfo,
) {
t.Helper()
@ -481,7 +483,7 @@ func runBackupAndCompare(
nameToID[ro] = ro
}
backupCtrl := loadController(ctx, t, config.Resource)
backupCtrl := newController(ctx, t, config.Resource)
backupCtrl.IDNameLookup = inMock.NewCache(idToName, nameToID)
backupSel := backupSelectorForExpected(t, config.Service, expectedDests)
@ -531,7 +533,7 @@ func runRestoreBackupTest(
ctx, flush := tester.NewContext(t)
defer flush()
config := ConfigInfo{
config := stub.ConfigInfo{
Opts: opts,
Resource: test.resourceCat,
Service: test.service,
@ -540,7 +542,7 @@ func runRestoreBackupTest(
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
totalItems, totalKopiaItems, collections, expectedData, err := GetCollectionsAndExpected(
totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected(
config,
test.collections,
version.Backup)
@ -576,16 +578,16 @@ func runRestoreTestWithVersion(
ctx, flush := tester.NewContext(t)
defer flush()
config := ConfigInfo{
config := stub.ConfigInfo{
Opts: opts,
Resource: test.resource,
Resource: test.resourceCat,
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
totalItems, _, collections, _, err := GetCollectionsAndExpected(
totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
config,
test.collectionsPrevious,
test.backupVersion)
@ -613,16 +615,16 @@ func runRestoreBackupTestVersions(
ctx, flush := tester.NewContext(t)
defer flush()
config := ConfigInfo{
config := stub.ConfigInfo{
Opts: opts,
Resource: test.resource,
Resource: test.resourceCat,
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
totalItems, _, collections, _, err := GetCollectionsAndExpected(
totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
config,
test.collectionsPrevious,
test.backupVersion)
@ -637,7 +639,7 @@ func runRestoreBackupTestVersions(
totalItems)
// Get expected output for new version.
totalItems, totalKopiaItems, _, expectedData, err := GetCollectionsAndExpected(
totalItems, totalKopiaItems, _, expectedData, err := stub.GetCollectionsAndExpected(
config,
test.collectionsLatest,
version.Backup)
@ -662,24 +664,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "EmailsWithAttachments",
service: path.ExchangeService,
resourceCat: resource.Users,
collections: []ColInfo{
collections: []stub.ColInfo{
{
PathElements: []string{"Inbox"},
Category: path.EmailCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID",
data: exchMock.MessageWithDirectAttachment(
Name: "someencodeditemID",
Data: exchMock.MessageWithDirectAttachment(
subjectText + "-1",
),
lookupKey: subjectText + "-1",
LookupKey: subjectText + "-1",
},
{
name: "someencodeditemID2",
data: exchMock.MessageWithTwoAttachments(
Name: "someencodeditemID2",
Data: exchMock.MessageWithTwoAttachments(
subjectText + "-2",
),
lookupKey: subjectText + "-2",
LookupKey: subjectText + "-2",
},
},
},
@ -689,73 +691,73 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleEmailsMultipleFolders",
service: path.ExchangeService,
resourceCat: resource.Users,
collections: []ColInfo{
collections: []stub.ColInfo{
{
PathElements: []string{"Inbox"},
Category: path.EmailCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID",
data: exchMock.MessageWithBodyBytes(
Name: "someencodeditemID",
Data: exchMock.MessageWithBodyBytes(
subjectText+"-1",
bodyText+" 1.",
bodyText+" 1.",
),
lookupKey: subjectText + "-1",
LookupKey: subjectText + "-1",
},
},
},
{
PathElements: []string{"Work"},
Category: path.EmailCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID2",
data: exchMock.MessageWithBodyBytes(
Name: "someencodeditemID2",
Data: exchMock.MessageWithBodyBytes(
subjectText+"-2",
bodyText+" 2.",
bodyText+" 2.",
),
lookupKey: subjectText + "-2",
LookupKey: subjectText + "-2",
},
{
name: "someencodeditemID3",
data: exchMock.MessageWithBodyBytes(
Name: "someencodeditemID3",
Data: exchMock.MessageWithBodyBytes(
subjectText+"-3",
bodyText+" 3.",
bodyText+" 3.",
),
lookupKey: subjectText + "-3",
LookupKey: subjectText + "-3",
},
},
},
{
PathElements: []string{"Work", "Inbox"},
Category: path.EmailCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID4",
data: exchMock.MessageWithBodyBytes(
Name: "someencodeditemID4",
Data: exchMock.MessageWithBodyBytes(
subjectText+"-4",
bodyText+" 4.",
bodyText+" 4.",
),
lookupKey: subjectText + "-4",
LookupKey: subjectText + "-4",
},
},
},
{
PathElements: []string{"Work", "Inbox", "Work"},
Category: path.EmailCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID5",
data: exchMock.MessageWithBodyBytes(
Name: "someencodeditemID5",
Data: exchMock.MessageWithBodyBytes(
subjectText+"-5",
bodyText+" 5.",
bodyText+" 5.",
),
lookupKey: subjectText + "-5",
LookupKey: subjectText + "-5",
},
},
},
@ -765,25 +767,25 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleContactsSingleFolder",
service: path.ExchangeService,
resourceCat: resource.Users,
collections: []ColInfo{
collections: []stub.ColInfo{
{
PathElements: []string{"Contacts"},
Category: path.ContactsCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"),
lookupKey: "Ghimley",
Name: "someencodeditemID",
Data: exchMock.ContactBytes("Ghimley"),
LookupKey: "Ghimley",
},
{
name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"),
lookupKey: "Irgot",
Name: "someencodeditemID2",
Data: exchMock.ContactBytes("Irgot"),
LookupKey: "Irgot",
},
{
name: "someencodeditemID3",
data: exchMock.ContactBytes("Jannes"),
lookupKey: "Jannes",
Name: "someencodeditemID3",
Data: exchMock.ContactBytes("Jannes"),
LookupKey: "Jannes",
},
},
},
@ -793,41 +795,41 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
name: "MultipleContactsMultipleFolders",
service: path.ExchangeService,
resourceCat: resource.Users,
collections: []ColInfo{
collections: []stub.ColInfo{
{
PathElements: []string{"Work"},
Category: path.ContactsCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"),
lookupKey: "Ghimley",
Name: "someencodeditemID",
Data: exchMock.ContactBytes("Ghimley"),
LookupKey: "Ghimley",
},
{
name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"),
lookupKey: "Irgot",
Name: "someencodeditemID2",
Data: exchMock.ContactBytes("Irgot"),
LookupKey: "Irgot",
},
{
name: "someencodeditemID3",
data: exchMock.ContactBytes("Jannes"),
lookupKey: "Jannes",
Name: "someencodeditemID3",
Data: exchMock.ContactBytes("Jannes"),
LookupKey: "Jannes",
},
},
},
{
PathElements: []string{"Personal"},
Category: path.ContactsCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID4",
data: exchMock.ContactBytes("Argon"),
lookupKey: "Argon",
Name: "someencodeditemID4",
Data: exchMock.ContactBytes("Argon"),
LookupKey: "Argon",
},
{
name: "someencodeditemID5",
data: exchMock.ContactBytes("Bernard"),
lookupKey: "Bernard",
Name: "someencodeditemID5",
Data: exchMock.ContactBytes("Bernard"),
LookupKey: "Bernard",
},
},
},
@ -926,26 +928,26 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
name: "Contacts",
service: path.ExchangeService,
resourceCat: resource.Users,
collections: []ColInfo{
collections: []stub.ColInfo{
{
PathElements: []string{"Work"},
Category: path.ContactsCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID",
data: exchMock.ContactBytes("Ghimley"),
lookupKey: "Ghimley",
Name: "someencodeditemID",
Data: exchMock.ContactBytes("Ghimley"),
LookupKey: "Ghimley",
},
},
},
{
PathElements: []string{"Personal"},
Category: path.ContactsCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "someencodeditemID2",
data: exchMock.ContactBytes("Irgot"),
lookupKey: "Irgot",
Name: "someencodeditemID2",
Data: exchMock.ContactBytes("Irgot"),
LookupKey: "Irgot",
},
},
},
@ -1004,12 +1006,12 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
},
})
totalItems, _, collections, expectedData, err := collectionsForInfo(
totalItems, _, collections, expectedData, err := stub.CollectionsForInfo(
test.service,
suite.ctrl.tenant,
suite.user,
restoreCfg,
[]ColInfo{collection},
[]stub.ColInfo{collection},
version.Backup,
)
require.NoError(t, err)
@ -1027,7 +1029,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
restoreCfg.Location,
)
restoreCtrl := loadController(ctx, t, test.resourceCat)
restoreCtrl := newController(ctx, t, test.resourceCat)
deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx,
version.Backup,
@ -1057,7 +1059,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
// Run a backup and compare its output with what we put in.
backupCtrl := loadController(ctx, t, test.resourceCat)
backupCtrl := newController(ctx, t, test.resourceCat)
backupSel := backupSelectorForExpected(t, test.service, expectedDests)
t.Log("Selective backup of", backupSel)
@ -1079,7 +1081,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
t.Log("Backup enumeration complete")
ci := ConfigInfo{
ci := stub.ConfigInfo{
Opts: control.Options{RestorePermissions: true},
// Alright to be empty, needed for OneDrive.
RestoreCfg: control.RestoreConfig{},
@ -1105,15 +1107,15 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
name: "EmailsWithLargeAttachments",
service: path.ExchangeService,
resourceCat: resource.Users,
collections: []ColInfo{
collections: []stub.ColInfo{
{
PathElements: []string{"Inbox"},
Category: path.EmailCategory,
Items: []ItemInfo{
Items: []stub.ItemInfo{
{
name: "35mbAttachment",
data: exchMock.MessageWithSizedAttachment(subjectText, 35),
lookupKey: subjectText,
Name: "35mbAttachment",
Data: exchMock.MessageWithSizedAttachment(subjectText, 35),
LookupKey: subjectText,
},
},
},
@ -1206,7 +1208,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
defer flush()
var (
backupCtrl = loadController(ctx, t, test.resourceCat)
backupCtrl = newController(ctx, t, test.resourceCat)
backupSel = test.selectorFunc(t)
errs = fault.New(true)
start = time.Now()
@ -1270,3 +1272,166 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
})
}
}
type DisconnectedUnitSuite struct {
tester.Suite
}
func TestDisconnectedUnitSuite(t *testing.T) {
s := &DisconnectedUnitSuite{
Suite: tester.NewUnitSuite(t),
}
suite.Run(t, s)
}
func statusTestTask(
t *testing.T,
ctrl *Controller,
objects, success, folder int,
) {
ctx, flush := tester.NewContext(t)
defer flush()
status := support.CreateStatus(
ctx,
support.Restore, folder,
support.CollectionMetrics{
Objects: objects,
Successes: success,
Bytes: 0,
},
"statusTestTask")
ctrl.UpdateStatus(status)
}
func (suite *DisconnectedUnitSuite) TestController_Status() {
t := suite.T()
ctrl := Controller{wg: &sync.WaitGroup{}}
// Two tasks
ctrl.incrementAwaitingMessages()
ctrl.incrementAwaitingMessages()
// Each helper task processes 4 objects, 1 success, 3 errors, 1 folders
go statusTestTask(t, &ctrl, 4, 1, 1)
go statusTestTask(t, &ctrl, 4, 1, 1)
stats := ctrl.Wait()
assert.NotEmpty(t, ctrl.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, stats.Objects)
// Expect 2 success
assert.Equal(t, 2, stats.Successes)
// Expect 2 folders
assert.Equal(t, 2, stats.Folders)
}
func (suite *DisconnectedUnitSuite) TestVerifyBackupInputs_allServices() {
sites := []string{"abc.site.foo", "bar.site.baz"}
tests := []struct {
name string
excludes func(t *testing.T) selectors.Selector
filters func(t *testing.T) selectors.Selector
includes func(t *testing.T) selectors.Selector
checkError assert.ErrorAssertionFunc
}{
{
name: "Valid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
},
{
name: "Invalid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
},
{
name: "valid sites",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Include(sel.AllData())
return sel.Selector
},
},
{
name: "invalid sites",
checkError: assert.Error,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Include(sel.AllData())
return sel.Selector
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
err := verifyBackupInputs(test.excludes(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.filters(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.includes(t), sites)
test.checkError(t, err, clues.ToCore(err))
})
}
}

0
src/internal/m365/exchange/contact_container_cache.go → src/internal/m365/exchange/contacts_container_cache.go
View File

0
src/internal/m365/exchange/event_container_cache.go → src/internal/m365/exchange/events_container_cache.go
View File

181
src/internal/m365/graph_connector_disconnected_test.go
View File

@ -1,181 +0,0 @@
package m365
import (
"sync"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
)
// ---------------------------------------------------------------
// Disconnected Test Section
// ---------------------------------------------------------------
type DisconnectedSuite struct {
tester.Suite
}
func TestSuite(t *testing.T) {
s := &DisconnectedSuite{
Suite: tester.NewUnitSuite(t),
}
suite.Run(t, s)
}
func statusTestTask(
t *testing.T,
ctrl *Controller,
objects, success, folder int,
) {
ctx, flush := tester.NewContext(t)
defer flush()
status := support.CreateStatus(
ctx,
support.Restore, folder,
support.CollectionMetrics{
Objects: objects,
Successes: success,
Bytes: 0,
},
"statusTestTask")
ctrl.UpdateStatus(status)
}
func (suite *DisconnectedSuite) TestController_Status() {
t := suite.T()
ctrl := Controller{wg: &sync.WaitGroup{}}
// Two tasks
ctrl.incrementAwaitingMessages()
ctrl.incrementAwaitingMessages()
// Each helper task processes 4 objects, 1 success, 3 errors, 1 folders
go statusTestTask(t, &ctrl, 4, 1, 1)
go statusTestTask(t, &ctrl, 4, 1, 1)
stats := ctrl.Wait()
assert.NotEmpty(t, ctrl.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, stats.Objects)
// Expect 2 success
assert.Equal(t, 2, stats.Successes)
// Expect 2 folders
assert.Equal(t, 2, stats.Folders)
}
func (suite *DisconnectedSuite) TestVerifyBackupInputs_allServices() {
sites := []string{"abc.site.foo", "bar.site.baz"}
tests := []struct {
name string
excludes func(t *testing.T) selectors.Selector
filters func(t *testing.T) selectors.Selector
includes func(t *testing.T) selectors.Selector
checkError assert.ErrorAssertionFunc
}{
{
name: "Valid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = "elliotReid@someHospital.org"
return sel.Selector
},
},
{
name: "Invalid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Exclude(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Filter(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
return sel.Selector
},
},
{
name: "valid sites",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
sel.Include(sel.AllData())
return sel.Selector
},
},
{
name: "invalid sites",
checkError: assert.Error,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Exclude(sel.AllData())
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Filter(sel.AllData())
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Include(sel.AllData())
return sel.Selector
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
err := verifyBackupInputs(test.excludes(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.filters(t), sites)
test.checkError(t, err, clues.ToCore(err))
err = verifyBackupInputs(test.includes(t), sites)
test.checkError(t, err, clues.ToCore(err))
})
}
}

26
src/internal/m365/graph_connector_helper_test.go → src/internal/m365/helper_test.go
View File

@ -19,7 +19,9 @@ import (
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/onedrive"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -102,15 +104,15 @@ func testElementsMatch[T any](
type restoreBackupInfo struct {
name string
service path.ServiceType
collections []ColInfo
collections []m365Stub.ColInfo
resourceCat resource.Category
}
type restoreBackupInfoMultiVersion struct {
service path.ServiceType
collectionsLatest []ColInfo
collectionsPrevious []ColInfo
resource resource.Category
collectionsLatest []m365Stub.ColInfo
collectionsPrevious []m365Stub.ColInfo
resourceCat resource.Category
backupVersion int
}
@ -686,7 +688,7 @@ func compareDriveItem(
t *testing.T,
expected map[string][]byte,
item data.Stream,
config ConfigInfo,
config m365Stub.ConfigInfo,
rootDir bool,
) bool {
// Skip Drive permissions in the folder that used to be the root. We don't
@ -793,7 +795,7 @@ func compareDriveItem(
return true
}
var fileData testOneDriveData
var fileData odStub.FileData
err = json.Unmarshal(buf, &fileData)
if !assert.NoError(t, err, "unmarshalling file data for file", name, clues.ToCore(err)) {
@ -829,7 +831,7 @@ func compareItem(
service path.ServiceType,
category path.CategoryType,
item data.Stream,
config ConfigInfo,
config m365Stub.ConfigInfo,
rootDir bool,
) bool {
if mt, ok := item.(data.StreamModTime); ok {
@ -923,7 +925,7 @@ func checkCollections(
expectedItems int,
expected map[string]map[string][]byte,
got []data.BackupCollection,
config ConfigInfo,
config m365Stub.ConfigInfo,
) int {
collectionsWithItems := []data.BackupCollection{}
@ -985,7 +987,7 @@ func checkCollections(
checkHasCollections(t, expected, collectionsWithItems)
// Return how many metadata files were skipped so we can account for it in the
// check on Controller status.
// check on controller status.
return skipped
}
@ -1152,11 +1154,11 @@ func getSelectorWith(
}
}
func loadController(ctx context.Context, t *testing.T, r resource.Category) *Controller {
func newController(ctx context.Context, t *testing.T, r resource.Category) *Controller {
a := tester.NewM365Account(t)
connector, err := NewController(ctx, a, r)
controller, err := NewController(ctx, a, r)
require.NoError(t, err, clues.ToCore(err))
return connector
return controller
}

24
src/internal/m365/mock/collection.go Normal file
View File

@ -0,0 +1,24 @@
package mock
import (
"context"
"github.com/alcionai/corso/src/internal/data"
)
type RestoreCollection struct {
data.Collection
AuxItems map[string]data.Stream
}
func (rc RestoreCollection) FetchItemByName(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.AuxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}

0
src/internal/m365/onedrive/drive.go → src/internal/m365/onedrive/item_collector.go
View File

7
src/internal/m365/onedrive/drive_test.go → src/internal/m365/onedrive/item_collector_test.go
View File

@ -28,13 +28,12 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api/mock"
)
// Unit tests
type OneDriveUnitSuite struct {
type ItemCollectorUnitSuite struct {
tester.Suite
}
func TestOneDriveUnitSuite(t *testing.T) {
suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)})
suite.Run(t, &ItemCollectorUnitSuite{Suite: tester.NewUnitSuite(t)})
}
const (
@ -51,7 +50,7 @@ func odErr(code string) *odataerrors.ODataError {
return odErr
}
func (suite *OneDriveUnitSuite) TestDrives() {
func (suite *ItemCollectorUnitSuite) TestDrives() {
t := suite.T()
ctx, flush := tester.NewContext(t)

174
src/internal/m365/graph_connector_onedrive_test_helper.go → src/internal/m365/onedrive/stub/stub.go
View File

@ -1,4 +1,4 @@
package m365
package stub
import (
"encoding/json"
@ -6,11 +6,10 @@ import (
"github.com/alcionai/clues"
"github.com/google/uuid"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/data"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/path"
)
@ -61,59 +60,59 @@ type ItemData struct {
Perms PermData
}
type OnedriveColInfo struct {
type ColInfo struct {
PathElements []string
Perms PermData
Files []ItemData
Folders []ItemData
}
type onedriveCollection struct {
service path.ServiceType
type collection struct {
Service path.ServiceType
PathElements []string
items []ItemInfo
aux []ItemInfo
backupVersion int
Items []m365Stub.ItemInfo
Aux []m365Stub.ItemInfo
BackupVersion int
}
func (c onedriveCollection) collection() ColInfo {
func (c collection) ColInfo() m365Stub.ColInfo {
cat := path.FilesCategory
if c.service == path.SharePointService {
if c.Service == path.SharePointService {
cat = path.LibrariesCategory
}
return ColInfo{
return m365Stub.ColInfo{
PathElements: c.PathElements,
Category: cat,
Items: c.items,
AuxItems: c.aux,
Items: c.Items,
AuxItems: c.Aux,
}
}
func NewOneDriveCollection(
func NewCollection(
service path.ServiceType,
PathElements []string,
backupVersion int,
) *onedriveCollection {
return &onedriveCollection{
service: service,
) *collection {
return &collection{
Service: service,
PathElements: PathElements,
backupVersion: backupVersion,
BackupVersion: backupVersion,
}
}
func DataForInfo(
service path.ServiceType,
cols []OnedriveColInfo,
cols []ColInfo,
backupVersion int,
) ([]ColInfo, error) {
) ([]m365Stub.ColInfo, error) {
var (
res []ColInfo
res []m365Stub.ColInfo
err error
)
for _, c := range cols {
onedriveCol := NewOneDriveCollection(service, c.PathElements, backupVersion)
onedriveCol := NewCollection(service, c.PathElements, backupVersion)
for _, f := range c.Files {
_, err = onedriveCol.withFile(f.Name, f.Data, f.Perms)
@ -134,18 +133,18 @@ func DataForInfo(
return res, err
}
res = append(res, onedriveCol.collection())
res = append(res, onedriveCol.ColInfo())
}
return res, nil
}
func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermData) (*onedriveCollection, error) {
switch c.backupVersion {
func (c *collection) withFile(name string, fileData []byte, perm PermData) (*collection, error) {
switch c.BackupVersion {
case 0:
// Lookups will occur using the most recent version of things so we need
// the embedded file name to match that.
item, err := onedriveItemWithData(
item, err := FileWithData(
name,
name+metadata.DataFileSuffix,
fileData)
@ -153,12 +152,12 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat
return c, err
}
c.items = append(c.items, item)
c.Items = append(c.Items, item)
// v1-5, early metadata design
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker,
version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName:
items, err := onedriveItemWithData(
items, err := FileWithData(
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData)
@ -166,24 +165,24 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat
return c, err
}
c.items = append(c.items, items)
c.Items = append(c.Items, items)
md, err := onedriveMetadata(
md, err := ItemWithMetadata(
"",
name+metadata.MetaFileSuffix,
name+metadata.MetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.BackupVersion >= versionPermissionSwitchedToID)
if err != nil {
return c, err
}
c.items = append(c.items, md)
c.aux = append(c.aux, md)
c.Items = append(c.Items, md)
c.Aux = append(c.Aux, md)
// v6+ current metadata design
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
item, err := onedriveItemWithData(
item, err := FileWithData(
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData)
@ -191,50 +190,50 @@ func (c *onedriveCollection) withFile(name string, fileData []byte, perm PermDat
return c, err
}
c.items = append(c.items, item)
c.Items = append(c.Items, item)
md, err := onedriveMetadata(
md, err := ItemWithMetadata(
name,
name+metadata.MetaFileSuffix,
name,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.BackupVersion >= versionPermissionSwitchedToID)
if err != nil {
return c, err
}
c.items = append(c.items, md)
c.aux = append(c.aux, md)
c.Items = append(c.Items, md)
c.Aux = append(c.Aux, md)
default:
return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.backupVersion))
return c, clues.New(fmt.Sprintf("bad backup version. version %d", c.BackupVersion))
}
return c, nil
}
func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCollection, error) {
switch c.backupVersion {
func (c *collection) withFolder(name string, perm PermData) (*collection, error) {
switch c.BackupVersion {
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
return c, nil
case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
item, err := onedriveMetadata(
item, err := ItemWithMetadata(
"",
name+metadata.DirMetaFileSuffix,
name+metadata.DirMetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.BackupVersion >= versionPermissionSwitchedToID)
c.items = append(c.items, item)
c.Items = append(c.Items, item)
if err != nil {
return c, err
}
default:
return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.backupVersion))
return c, clues.New(fmt.Sprintf("bad backup version.version %d", c.BackupVersion))
}
return c, nil
@ -242,17 +241,17 @@ func (c *onedriveCollection) withFolder(name string, perm PermData) (*onedriveCo
// withPermissions adds permissions to the folder represented by this
// onedriveCollection.
func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection, error) {
func (c *collection) withPermissions(perm PermData) (*collection, error) {
// These versions didn't store permissions for the folder or didn't store them
// in the folder's collection.
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
if c.BackupVersion < version.OneDrive4DirIncludesPermissions {
return c, nil
}
name := c.PathElements[len(c.PathElements)-1]
metaName := name
if c.backupVersion >= version.OneDrive5DirMetaNoName {
if c.BackupVersion >= version.OneDrive5DirMetaNoName {
// We switched to just .dirmeta for metadata file names.
metaName = ""
}
@ -261,98 +260,63 @@ func (c *onedriveCollection) withPermissions(perm PermData) (*onedriveCollection
return c, nil
}
md, err := onedriveMetadata(
md, err := ItemWithMetadata(
name,
metaName+metadata.DirMetaFileSuffix,
metaName+metadata.DirMetaFileSuffix,
perm,
c.backupVersion >= versionPermissionSwitchedToID)
c.BackupVersion >= versionPermissionSwitchedToID)
if err != nil {
return c, err
}
c.items = append(c.items, md)
c.aux = append(c.aux, md)
c.Items = append(c.Items, md)
c.Aux = append(c.Aux, md)
return c, err
}
type testOneDriveData struct {
type FileData struct {
FileName string `json:"fileName,omitempty"`
Data []byte `json:"data,omitempty"`
}
func onedriveItemWithData(
func FileWithData(
name, lookupKey string,
fileData []byte,
) (ItemInfo, error) {
content := testOneDriveData{
) (m365Stub.ItemInfo, error) {
content := FileData{
FileName: lookupKey,
Data: fileData,
}
serialized, err := json.Marshal(content)
if err != nil {
return ItemInfo{}, clues.Stack(err)
return m365Stub.ItemInfo{}, clues.Stack(err)
}
return ItemInfo{
name: name,
data: serialized,
lookupKey: lookupKey,
return m365Stub.ItemInfo{
Name: name,
Data: serialized,
LookupKey: lookupKey,
}, nil
}
func onedriveMetadata(
func ItemWithMetadata(
fileName, itemID, lookupKey string,
perm PermData,
permUseID bool,
) (ItemInfo, error) {
) (m365Stub.ItemInfo, error) {
testMeta := getMetadata(fileName, perm, permUseID)
testMetaJSON, err := json.Marshal(testMeta)
if err != nil {
return ItemInfo{}, clues.Wrap(err, "marshalling metadata")
return m365Stub.ItemInfo{}, clues.Wrap(err, "marshalling metadata")
}
return ItemInfo{
name: itemID,
data: testMetaJSON,
lookupKey: lookupKey,
return m365Stub.ItemInfo{
Name: itemID,
Data: testMetaJSON,
LookupKey: lookupKey,
}, nil
}
func GetCollectionsAndExpected(
config ConfigInfo,
testCollections []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections []data.RestoreCollection
expectedData = map[string]map[string][]byte{}
totalItems = 0
totalKopiaItems = 0
)
for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData, err := collectionsForInfo(
config.Service,
config.Tenant,
owner,
config.RestoreCfg,
testCollections,
backupVersion,
)
if err != nil {
return totalItems, totalKopiaItems, collections, expectedData, err
}
collections = append(collections, ownerCollections...)
totalItems += numItems
totalKopiaItems += kopiaItems
maps.Copy(expectedData, userExpectedData)
}
return totalItems, totalKopiaItems, collections, expectedData, nil
}

145
src/internal/m365/graph_connector_onedrive_test.go → src/internal/m365/onedrive_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/graph"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
@ -99,7 +100,7 @@ type suiteInfoImpl struct {
ac api.Client
controller *Controller
resourceOwner string
resourceCat resource.Category
resourceCategory resource.Category
secondaryUser string
secondaryUserID string
service path.ServiceType
@ -115,18 +116,18 @@ func NewSuiteInfoImpl(
resourceOwner string,
service path.ServiceType,
) suiteInfoImpl {
rc := resource.Users
rsc := resource.Users
if service == path.SharePointService {
rc = resource.Sites
rsc = resource.Sites
}
ctrl := loadController(ctx, t, rc)
ctrl := newController(ctx, t, rsc)
return suiteInfoImpl{
ac: ctrl.AC,
controller: ctrl,
resourceOwner: resourceOwner,
resourceCat: rc,
resourceCategory: rsc,
secondaryUser: tester.SecondaryM365UserID(t),
service: service,
tertiaryUser: tester.TertiaryM365UserID(t),
@ -163,7 +164,7 @@ func (si suiteInfoImpl) Service() path.ServiceType {
}
func (si suiteInfoImpl) Resource() resource.Category {
return si.resourceCat
return si.resourceCategory
}
// ---------------------------------------------------------------------------
@ -402,16 +403,16 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
folderBName,
}
cols := []OnedriveColInfo{
cols := []stub.ColInfo{
{
PathElements: rootPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileAData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderAName,
},
@ -422,13 +423,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
},
{
PathElements: folderAPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileBData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderBName,
},
@ -436,13 +437,13 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
},
{
PathElements: subfolderBPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileCData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderAName,
},
@ -450,7 +451,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
},
{
PathElements: subfolderAPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileDData,
@ -459,7 +460,7 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
},
{
PathElements: folderBPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileEData,
@ -468,18 +469,18 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
},
}
expected, err := DataForInfo(suite.Service(), cols, version.Backup)
expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err)
for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("Version%d", vn), func() {
t := suite.T()
input, err := DataForInfo(suite.Service(), cols, vn)
input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{
service: suite.Service(),
resource: suite.Resource(),
resourceCat: suite.Resource(),
backupVersion: vn,
collectionsPrevious: input,
collectionsLatest: expected,
@ -549,15 +550,15 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
folderCName,
}
cols := []OnedriveColInfo{
cols := []stub.ColInfo{
{
PathElements: rootPath,
Files: []ItemData{
Files: []stub.ItemData{
{
// Test restoring a file that doesn't inherit permissions.
Name: fileName,
Data: fileAData,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
@ -570,13 +571,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
Data: fileBData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderBName,
},
{
Name: folderAName,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -584,7 +585,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
},
{
Name: folderCName,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -594,23 +595,23 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
},
{
PathElements: folderBPath,
Files: []ItemData{
Files: []stub.ItemData{
{
// Test restoring a file in a non-root folder that doesn't inherit
// permissions.
Name: fileName,
Data: fileBData,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
},
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderAName,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -624,18 +625,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// // Tests a folder that has permissions with an item in the folder with
// // the same permissions.
// pathElements: subfolderAPath,
// files: []itemData{
// files: []stub.ItemData{
// {
// name: fileName,
// data: fileDData,
// perms: permData{
// perms: stub.PermData{
// user: secondaryUserName,
// entityID: secondaryUserID,
// roles: readPerm,
// },
// },
// },
// Perms: PermData{
// Perms: stub.PermData{
// User: secondaryUserName,
// EntityID: secondaryUserID,
// Roles: readPerm,
@ -645,18 +646,18 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Tests a folder that has permissions with an item in the folder with
// the different permissions.
PathElements: folderAPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileEData,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
},
},
},
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -666,13 +667,13 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Tests a folder that has permissions with an item in the folder with
// no permissions.
PathElements: folderCPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileAData,
},
},
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: readPerm,
@ -680,7 +681,7 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
},
}
expected, err := DataForInfo(suite.Service(), cols, version.Backup)
expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err)
bss := suite.Service().String()
@ -690,12 +691,12 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
// Ideally this can always be true or false and still
// work, but limiting older versions to use emails so as
// to validate that flow as well.
input, err := DataForInfo(suite.Service(), cols, vn)
input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{
service: suite.Service(),
resource: suite.Resource(),
resourceCat: suite.Resource(),
backupVersion: vn,
collectionsPrevious: input,
collectionsLatest: expected,
@ -730,18 +731,18 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
suite.Service(),
suite.ResourceOwner())
inputCols := []OnedriveColInfo{
inputCols := []stub.ColInfo{
{
PathElements: []string{
odConsts.DrivesPathDir,
driveID,
odConsts.RootPathDir,
},
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileAData,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
@ -751,14 +752,14 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
},
}
expectedCols := []OnedriveColInfo{
expectedCols := []stub.ColInfo{
{
PathElements: []string{
odConsts.DrivesPathDir,
driveID,
odConsts.RootPathDir,
},
Files: []ItemData{
Files: []stub.ItemData{
{
// No permissions on the output since they weren't restored.
Name: fileName,
@ -768,19 +769,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
},
}
expected, err := DataForInfo(suite.Service(), expectedCols, version.Backup)
expected, err := stub.DataForInfo(suite.Service(), expectedCols, version.Backup)
require.NoError(suite.T(), err)
bss := suite.Service().String()
for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() {
t := suite.T()
input, err := DataForInfo(suite.Service(), inputCols, vn)
input, err := stub.DataForInfo(suite.Service(), inputCols, vn)
require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{
service: suite.Service(),
resource: suite.Resource(),
resourceCat: suite.Resource(),
backupVersion: vn,
collectionsPrevious: input,
collectionsLatest: expected,
@ -855,11 +856,11 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
folderCName,
}
fileSet := []ItemData{
fileSet := []stub.ItemData{
{
Name: "file-custom",
Data: fileAData,
Perms: PermData{
Perms: stub.PermData{
User: secondaryUserName,
EntityID: secondaryUserID,
Roles: writePerm,
@ -869,14 +870,14 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
{
Name: "file-inherited",
Data: fileAData,
Perms: PermData{
Perms: stub.PermData{
SharingMode: metadata.SharingModeInherited,
},
},
{
Name: "file-empty",
Data: fileAData,
Perms: PermData{
Perms: stub.PermData{
SharingMode: metadata.SharingModeCustom,
},
},
@ -900,23 +901,23 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
// - inherted-permission-file
// - empty-permission-file (empty/empty might have interesting behavior)
cols := []OnedriveColInfo{
cols := []stub.ColInfo{
{
PathElements: rootPath,
Files: []ItemData{},
Folders: []ItemData{
Files: []stub.ItemData{},
Folders: []stub.ItemData{
{Name: folderAName},
},
},
{
PathElements: folderAPath,
Files: fileSet,
Folders: []ItemData{
Folders: []stub.ItemData{
{Name: folderAName},
{Name: folderBName},
{Name: folderCName},
},
Perms: PermData{
Perms: stub.PermData{
User: tertiaryUserName,
EntityID: tertiaryUserID,
Roles: readPerm,
@ -925,7 +926,7 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
{
PathElements: subfolderAAPath,
Files: fileSet,
Perms: PermData{
Perms: stub.PermData{
User: tertiaryUserName,
EntityID: tertiaryUserID,
Roles: writePerm,
@ -935,20 +936,20 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
{
PathElements: subfolderABPath,
Files: fileSet,
Perms: PermData{
Perms: stub.PermData{
SharingMode: metadata.SharingModeInherited,
},
},
{
PathElements: subfolderACPath,
Files: fileSet,
Perms: PermData{
Perms: stub.PermData{
SharingMode: metadata.SharingModeCustom,
},
},
}
expected, err := DataForInfo(suite.Service(), cols, version.Backup)
expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err)
bss := suite.Service().String()
@ -958,12 +959,12 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
// Ideally this can always be true or false and still
// work, but limiting older versions to use emails so as
// to validate that flow as well.
input, err := DataForInfo(suite.Service(), cols, vn)
input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{
service: suite.Service(),
resource: suite.Resource(),
resourceCat: suite.Resource(),
backupVersion: vn,
collectionsPrevious: input,
collectionsLatest: expected,
@ -1018,16 +1019,16 @@ func testRestoreFolderNamedFolderRegression(
folderBName,
}
cols := []OnedriveColInfo{
cols := []stub.ColInfo{
{
PathElements: rootPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileAData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderNamedFolder,
},
@ -1038,13 +1039,13 @@ func testRestoreFolderNamedFolderRegression(
},
{
PathElements: folderFolderPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileBData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderBName,
},
@ -1052,13 +1053,13 @@ func testRestoreFolderNamedFolderRegression(
},
{
PathElements: subfolderPath,
Files: []ItemData{
Files: []stub.ItemData{
{
Name: fileName,
Data: fileCData,
},
},
Folders: []ItemData{
Folders: []stub.ItemData{
{
Name: folderNamedFolder,
},
@ -1066,19 +1067,19 @@ func testRestoreFolderNamedFolderRegression(
},
}
expected, err := DataForInfo(suite.Service(), cols, version.Backup)
expected, err := stub.DataForInfo(suite.Service(), cols, version.Backup)
require.NoError(suite.T(), err)
bss := suite.Service().String()
for vn := startVersion; vn <= version.Backup; vn++ {
suite.Run(fmt.Sprintf("%s-Version%d", bss, vn), func() {
t := suite.T()
input, err := DataForInfo(suite.Service(), cols, vn)
input, err := stub.DataForInfo(suite.Service(), cols, vn)
require.NoError(suite.T(), err)
testData := restoreBackupInfoMultiVersion{
service: suite.Service(),
resource: suite.Resource(),
resourceCat: suite.Resource(),
backupVersion: vn,
collectionsPrevious: input,
collectionsLatest: expected,

2
src/internal/m365/sharepoint/api/pages.go
View File

@ -172,7 +172,7 @@ func RestoreSitePage(
itemData data.Stream,
siteID, destName string,
) (details.ItemInfo, error) {
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID()))
ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePage", diagnostics.Label("item_uuid", itemData.UUID()))
defer end()
var (

4
src/internal/m365/sharepoint/collection.go
View File

@ -257,7 +257,7 @@ func (sc *Collection) retrieveLists(
sc.data <- &Item{
id: ptr.Val(lst.GetId()),
data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointListInfo(lst, size),
info: listToSPInfo(lst, size),
modTime: t,
}
@ -320,7 +320,7 @@ func (sc *Collection) retrievePages(
sc.data <- &Item{
id: ptr.Val(pg.GetId()),
data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointPageInfo(pg, root, size),
info: pageToSPInfo(pg, root, size),
modTime: ptr.OrNow(pg.GetLastModifiedDateTime()),
}

4
src/internal/m365/sharepoint/collection_test.go
View File

@ -116,7 +116,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
data := &Item{
id: name,
data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointListInfo(listing, int64(len(byteArray))),
info: listToSPInfo(listing, int64(len(byteArray))),
}
return data
@ -205,7 +205,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
listData := &Item{
id: testName,
data: io.NopCloser(bytes.NewReader(byteArray)),
info: sharePointListInfo(listing, int64(len(byteArray))),
info: listToSPInfo(listing, int64(len(byteArray))),
}
destName := tester.DefaultTestRestoreConfig("").Location

28
src/internal/m365/sharepoint/list_info.go
View File

@ -1,28 +0,0 @@
package sharepoint
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/backup/details"
)
// sharePointListInfo translates models.Listable metadata into searchable content
// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
func sharePointListInfo(lst models.Listable, size int64) *details.SharePointInfo {
var (
name = ptr.Val(lst.GetDisplayName())
webURL = ptr.Val(lst.GetWebUrl())
created = ptr.Val(lst.GetCreatedDateTime())
modified = ptr.Val(lst.GetLastModifiedDateTime())
)
return &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: name,
Created: created,
Modified: modified,
WebURL: webURL,
Size: size,
}
}

59
src/internal/m365/sharepoint/list_info_test.go
View File

@ -1,59 +0,0 @@
package sharepoint
import (
"testing"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
)
type SharePointInfoSuite struct {
tester.Suite
}
func TestSharePointInfoSuite(t *testing.T) {
suite.Run(t, &SharePointInfoSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *SharePointInfoSuite) TestSharePointInfo() {
tests := []struct {
name string
listAndDeets func() (models.Listable, *details.SharePointInfo)
}{
{
name: "Empty List",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
i := &details.SharePointInfo{ItemType: details.SharePointList}
return models.NewList(), i
},
}, {
name: "Only Name",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
aTitle := "Whole List"
listing := models.NewList()
listing.SetDisplayName(&aTitle)
i := &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: aTitle,
}
return listing, i
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
list, expected := test.listAndDeets()
info := sharePointListInfo(list, 10)
assert.Equal(t, expected.ItemType, info.ItemType)
assert.Equal(t, expected.ItemName, info.ItemName)
assert.Equal(t, expected.WebURL, info.WebURL)
})
}
}

21
src/internal/m365/sharepoint/list.go → src/internal/m365/sharepoint/lists.go
View File

@ -10,9 +10,30 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
)
// listToSPInfo translates models.Listable metadata into searchable content
// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
func listToSPInfo(lst models.Listable, size int64) *details.SharePointInfo {
var (
name = ptr.Val(lst.GetDisplayName())
webURL = ptr.Val(lst.GetWebUrl())
created = ptr.Val(lst.GetCreatedDateTime())
modified = ptr.Val(lst.GetLastModifiedDateTime())
)
return &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: name,
Created: created,
Modified: modified,
WebURL: webURL,
Size: size,
}
}
type listTuple struct {
name string
id string

51
src/internal/m365/sharepoint/list_test.go → src/internal/m365/sharepoint/lists_test.go
View File

@ -4,21 +4,23 @@ import (
"testing"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
)
type SharePointSuite struct {
type ListsUnitSuite struct {
tester.Suite
creds account.M365Config
}
func (suite *SharePointSuite) SetupSuite() {
func (suite *ListsUnitSuite) SetupSuite() {
t := suite.T()
a := tester.NewM365Account(t)
m365, err := a.M365Config()
@ -27,8 +29,8 @@ func (suite *SharePointSuite) SetupSuite() {
suite.creds = m365
}
func TestSharePointSuite(t *testing.T) {
suite.Run(t, &SharePointSuite{
func TestListsUnitSuite(t *testing.T) {
suite.Run(t, &ListsUnitSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tester.M365AcctCredEnvs},
@ -47,7 +49,7 @@ func TestSharePointSuite(t *testing.T) {
// to verify if these 2 calls are valid
// - fetchContentBaseTypes
// - fetchColumnPositions
func (suite *SharePointSuite) TestLoadList() {
func (suite *ListsUnitSuite) TestLoadList() {
t := suite.T()
ctx, flush := tester.NewContext(t)
@ -63,3 +65,42 @@ func (suite *SharePointSuite) TestLoadList() {
assert.Greater(t, len(lists), 0)
t.Logf("Length: %d\n", len(lists))
}
func (suite *ListsUnitSuite) TestSharePointInfo() {
tests := []struct {
name string
listAndDeets func() (models.Listable, *details.SharePointInfo)
}{
{
name: "Empty List",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
i := &details.SharePointInfo{ItemType: details.SharePointList}
return models.NewList(), i
},
}, {
name: "Only Name",
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
aTitle := "Whole List"
listing := models.NewList()
listing.SetDisplayName(&aTitle)
i := &details.SharePointInfo{
ItemType: details.SharePointList,
ItemName: aTitle,
}
return listing, i
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
list, expected := test.listAndDeets()
info := listToSPInfo(list, 10)
assert.Equal(t, expected.ItemType, info.ItemType)
assert.Equal(t, expected.ItemName, info.ItemName)
assert.Equal(t, expected.WebURL, info.WebURL)
})
}
}

4
src/internal/m365/sharepoint/pageInfo.go → src/internal/m365/sharepoint/pages.go
View File

@ -8,10 +8,10 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
)
// sharePointPageInfo propagates metadata from the SharePoint Page data type
// pageToSPInfo propagates metadata from the SharePoint Page data type
// into searchable content.
// Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
func sharePointPageInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo {
func pageToSPInfo(page models.SitePageable, root string, size int64) *details.SharePointInfo {
var (
name, prefix, webURL string
created, modified time.Time

16
src/internal/m365/sharepoint/pageInfo_test.go → src/internal/m365/sharepoint/pages_test.go
View File

@ -1,13 +1,25 @@
package sharepoint
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/m365/graph/betasdk/models"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
)
func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() {
type PagesUnitSuite struct {
tester.Suite
}
func TestPagesUnitSuite(t *testing.T) {
suite.Run(t, &PagesUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *PagesUnitSuite) TestSharePointInfo_Pages() {
tests := []struct {
name string
pageAndDeets func() (models.SitePageable, *details.SharePointInfo)
@ -39,7 +51,7 @@ func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() {
t := suite.T()
paged, expected := test.pageAndDeets()
info := sharePointPageInfo(paged, "", 0)
info := pageToSPInfo(paged, "", 0)
assert.Equal(t, expected.ItemType, info.ItemType)
assert.Equal(t, expected.ItemName, info.ItemName)
assert.Equal(t, expected.WebURL, info.WebURL)

12
src/internal/m365/sharepoint/restore.go
View File

@ -129,7 +129,7 @@ func restoreListItem(
itemData data.Stream,
siteID, destName string,
) (details.ItemInfo, error) {
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID()))
ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.UUID()))
defer end()
ctx = clues.Add(ctx, "list_item_id", itemData.UUID())
@ -190,7 +190,7 @@ func restoreListItem(
}
}
dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray)))
dii.SharePoint = listToSPInfo(restoredList, int64(len(byteArray)))
return dii, nil
}
@ -203,7 +203,7 @@ func RestoreListCollection(
deets *details.Builder,
errs *fault.Bus,
) (support.CollectionMetrics, error) {
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath()))
ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath()))
defer end()
var (
@ -214,7 +214,7 @@ func RestoreListCollection(
el = errs.Local()
)
trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String())
trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String())
for {
if el.Failure() != nil {
@ -285,8 +285,8 @@ func RestorePageCollection(
siteID = directory.ResourceOwner()
)
trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
ctx, end := diagnostics.Span(ctx, "gc:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath()))
trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String())
ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restorePageCollection", diagnostics.Label("path", dc.FullPath()))
defer end()

223
src/internal/m365/graph_connector_test_helper.go → src/internal/m365/stub/stub.go
View File

@ -1,12 +1,14 @@
package m365
package stub
import (
"bytes"
"context"
"io"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/data"
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
"github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/pkg/control"
@ -33,9 +35,9 @@ type ItemInfo struct {
// be the same before and after restoring the item in M365 and may not be
// the M365 ID. When restoring items out of place, the item is assigned a
// new ID making it unsuitable for a lookup key.
lookupKey string
name string
data []byte
LookupKey string
Name string
Data []byte
}
type ConfigInfo struct {
@ -47,19 +49,113 @@ type ConfigInfo struct {
RestoreCfg control.RestoreConfig
}
func mustToDataLayerPath(
service path.ServiceType,
tenant, resourceOwner string,
category path.CategoryType,
elements []string,
isItem bool,
) (path.Path, error) {
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
func GetCollectionsAndExpected(
config ConfigInfo,
testCollections []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections []data.RestoreCollection
expectedData = map[string]map[string][]byte{}
totalItems = 0
totalKopiaItems = 0
)
for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo(
config.Service,
config.Tenant,
owner,
config.RestoreCfg,
testCollections,
backupVersion,
)
if err != nil {
return nil, err
return totalItems, totalKopiaItems, collections, expectedData, err
}
return res, err
collections = append(collections, ownerCollections...)
totalItems += numItems
totalKopiaItems += kopiaItems
maps.Copy(expectedData, userExpectedData)
}
return totalItems, totalKopiaItems, collections, expectedData, nil
}
func CollectionsForInfo(
service path.ServiceType,
tenant, user string,
restoreCfg control.RestoreConfig,
allInfo []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections = make([]data.RestoreCollection, 0, len(allInfo))
expectedData = make(map[string]map[string][]byte, len(allInfo))
totalItems = 0
kopiaEntries = 0
)
for _, info := range allInfo {
pth, err := path.Build(
tenant,
user,
service,
info.Category,
false,
info.PathElements...)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
mc := exchMock.NewCollection(pth, pth, len(info.Items))
baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
baseExpected := expectedData[baseDestPath.String()]
if baseExpected == nil {
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items))
baseExpected = expectedData[baseDestPath.String()]
}
for i := 0; i < len(info.Items); i++ {
mc.Names[i] = info.Items[i].Name
mc.Data[i] = info.Items[i].Data
baseExpected[info.Items[i].LookupKey] = info.Items[i].Data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].Name) {
continue
}
totalItems++
}
c := mock.RestoreCollection{
Collection: mc,
AuxItems: map[string]data.Stream{},
}
for _, aux := range info.AuxItems {
c.AuxItems[aux.Name] = &exchMock.Data{
ID: aux.Name,
Reader: io.NopCloser(bytes.NewReader(aux.Data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.Items)
}
return totalItems, kopiaEntries, collections, expectedData, nil
}
// backupOutputPathFromRestore returns a path.Path denoting the location in
@ -86,102 +182,11 @@ func backupOutputPathFromRestore(
base = append(base, inputPath.Folders()...)
}
return mustToDataLayerPath(
inputPath.Service(),
return path.Build(
inputPath.Tenant(),
inputPath.ResourceOwner(),
inputPath.Service(),
inputPath.Category(),
base,
false,
)
}
// TODO(ashmrtn): Make this an actual mock class that can be used in other
// packages.
type mockRestoreCollection struct {
data.Collection
auxItems map[string]data.Stream
}
func (rc mockRestoreCollection) FetchItemByName(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.auxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}
func collectionsForInfo(
service path.ServiceType,
tenant, user string,
restoreCfg control.RestoreConfig,
allInfo []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
var (
collections = make([]data.RestoreCollection, 0, len(allInfo))
expectedData = make(map[string]map[string][]byte, len(allInfo))
totalItems = 0
kopiaEntries = 0
)
for _, info := range allInfo {
pth, err := mustToDataLayerPath(
service,
tenant,
user,
info.Category,
info.PathElements,
false)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
mc := exchMock.NewCollection(pth, pth, len(info.Items))
baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}
baseExpected := expectedData[baseDestPath.String()]
if baseExpected == nil {
expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.Items))
baseExpected = expectedData[baseDestPath.String()]
}
for i := 0; i < len(info.Items); i++ {
mc.Names[i] = info.Items[i].name
mc.Data[i] = info.Items[i].data
baseExpected[info.Items[i].lookupKey] = info.Items[i].data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].name) {
continue
}
totalItems++
}
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.AuxItems {
c.auxItems[aux.name] = &exchMock.Data{
ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.Items)
}
return totalItems, kopiaEntries, collections, expectedData, nil
base...)
}