8e080f83b7
Exchange tests inadvertently got disabled since it wasn't finding path matches for returned BackupCollections. This switches to using LocationPath which does allow for matching Most contacts tests are disabled since restore doesn't support nested folders --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [ ] ⛔ No #### Type of change - [ ] 🌻 Feature - [x] 🐛 Bugfix - [ ] 🗺️ Documentation - [x] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Test Plan - [x] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
577 lines
16 KiB
Go
577 lines
16 KiB
Go
package m365
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"testing"
|
|
|
|
"github.com/alcionai/clues"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/stretchr/testify/suite"
|
|
|
|
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
|
|
"github.com/alcionai/corso/src/internal/m365/service/exchange"
|
|
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
|
|
"github.com/alcionai/corso/src/internal/m365/service/sharepoint"
|
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
|
"github.com/alcionai/corso/src/internal/tester"
|
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
|
"github.com/alcionai/corso/src/internal/version"
|
|
"github.com/alcionai/corso/src/pkg/control"
|
|
"github.com/alcionai/corso/src/pkg/fault"
|
|
"github.com/alcionai/corso/src/pkg/path"
|
|
"github.com/alcionai/corso/src/pkg/selectors"
|
|
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// DataCollection tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type DataCollectionIntgSuite struct {
|
|
tester.Suite
|
|
user string
|
|
site string
|
|
tenantID string
|
|
ac api.Client
|
|
}
|
|
|
|
func TestDataCollectionIntgSuite(t *testing.T) {
|
|
suite.Run(t, &DataCollectionIntgSuite{
|
|
Suite: tester.NewIntegrationSuite(
|
|
t,
|
|
[][]string{tconfig.M365AcctCredEnvs}),
|
|
})
|
|
}
|
|
|
|
func (suite *DataCollectionIntgSuite) SetupSuite() {
|
|
t := suite.T()
|
|
|
|
suite.user = tconfig.M365UserID(t)
|
|
suite.site = tconfig.M365SiteID(t)
|
|
|
|
acct := tconfig.NewM365Account(t)
|
|
creds, err := acct.M365Config()
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
suite.tenantID = creds.AzureTenantID
|
|
|
|
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
}
|
|
|
|
func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() {
|
|
ctx, flush := tester.NewContext(suite.T())
|
|
defer flush()
|
|
|
|
selUsers := []string{suite.user}
|
|
|
|
ctrl := newController(ctx, suite.T(), path.ExchangeService)
|
|
tests := []struct {
|
|
name string
|
|
getSelector func(t *testing.T) selectors.Selector
|
|
}{
|
|
{
|
|
name: "Email",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewExchangeBackup(selUsers)
|
|
sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
|
sel.DiscreteOwner = suite.user
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "Contacts",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewExchangeBackup(selUsers)
|
|
sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()))
|
|
sel.DiscreteOwner = suite.user
|
|
return sel.Selector
|
|
},
|
|
},
|
|
// {
|
|
// name: "Events",
|
|
// getSelector: func(t *testing.T) selectors.Selector {
|
|
// sel := selectors.NewExchangeBackup(selUsers)
|
|
// sel.Include(sel.EventCalendars([]string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
|
|
// sel.DiscreteOwner = suite.user
|
|
// return sel.Selector
|
|
// },
|
|
// },
|
|
}
|
|
|
|
for _, test := range tests {
|
|
for _, canMakeDeltaQueries := range []bool{true, false} {
|
|
name := test.name
|
|
|
|
if canMakeDeltaQueries {
|
|
name += "-delta"
|
|
} else {
|
|
name += "-non-delta"
|
|
}
|
|
|
|
suite.Run(name, func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
sel := test.getSelector(t)
|
|
uidn := inMock.NewProvider(sel.ID(), sel.Name())
|
|
|
|
ctrlOpts := control.DefaultOptions()
|
|
ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries
|
|
|
|
bpc := inject.BackupProducerConfig{
|
|
// exchange doesn't have any changes based on backup version yet.
|
|
LastBackupVersion: version.NoBackup,
|
|
Options: ctrlOpts,
|
|
ProtectedResource: uidn,
|
|
Selector: sel,
|
|
}
|
|
|
|
collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections(
|
|
ctx,
|
|
bpc,
|
|
suite.ac,
|
|
suite.tenantID,
|
|
ctrl.UpdateStatus,
|
|
fault.New(true))
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
|
assert.True(t, excludes.Empty())
|
|
|
|
for range collections {
|
|
ctrl.incrementAwaitingMessages()
|
|
}
|
|
|
|
// Categories with delta endpoints will produce a collection for metadata
|
|
// as well as the actual data pulled, and the "temp" root collection.
|
|
assert.LessOrEqual(t, 1, len(collections), "expected 1 <= num collections <= 3")
|
|
assert.GreaterOrEqual(t, 3, len(collections), "expected 1 <= num collections <= 3")
|
|
|
|
for _, col := range collections {
|
|
for object := range col.Items(ctx, fault.New(true)) {
|
|
buf := &bytes.Buffer{}
|
|
_, err := buf.ReadFrom(object.ToReader())
|
|
assert.NoError(t, err, "received a buf.Read error", clues.ToCore(err))
|
|
}
|
|
}
|
|
|
|
status := ctrl.Wait()
|
|
assert.NotZero(t, status.Successes)
|
|
t.Log(status.String())
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestInvalidUserForDataCollections ensures verification process for users
|
|
func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() {
|
|
ctx, flush := tester.NewContext(suite.T())
|
|
defer flush()
|
|
|
|
owners := []string{"snuffleupagus"}
|
|
ctrl := newController(ctx, suite.T(), path.ExchangeService)
|
|
tests := []struct {
|
|
name string
|
|
getSelector func(t *testing.T) selectors.Selector
|
|
}{
|
|
{
|
|
name: "invalid exchange backup user",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewExchangeBackup(owners)
|
|
sel.Include(sel.MailFolders(selectors.Any()))
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "Invalid onedrive backup user",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewOneDriveBackup(owners)
|
|
sel.Include(selTD.OneDriveBackupFolderScope(sel))
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "Invalid sharepoint backup site",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewSharePointBackup(owners)
|
|
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "missing exchange backup user",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewExchangeBackup(owners)
|
|
sel.Include(sel.MailFolders(selectors.Any()))
|
|
sel.DiscreteOwner = ""
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "missing onedrive backup user",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewOneDriveBackup(owners)
|
|
sel.Include(selTD.OneDriveBackupFolderScope(sel))
|
|
sel.DiscreteOwner = ""
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "missing sharepoint backup site",
|
|
getSelector: func(t *testing.T) selectors.Selector {
|
|
sel := selectors.NewSharePointBackup(owners)
|
|
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
|
sel.DiscreteOwner = ""
|
|
return sel.Selector
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
suite.Run(test.name, func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
bpc := inject.BackupProducerConfig{
|
|
LastBackupVersion: version.NoBackup,
|
|
Options: control.DefaultOptions(),
|
|
ProtectedResource: test.getSelector(t),
|
|
}
|
|
|
|
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
|
ctx,
|
|
bpc,
|
|
fault.New(true))
|
|
assert.Error(t, err, clues.ToCore(err))
|
|
assert.False(t, canUsePreviousBackup, "can use previous backup")
|
|
assert.Empty(t, collections)
|
|
assert.Nil(t, excludes)
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
|
|
ctx, flush := tester.NewContext(suite.T())
|
|
defer flush()
|
|
|
|
selSites := []string{suite.site}
|
|
ctrl := newController(ctx, suite.T(), path.SharePointService)
|
|
tests := []struct {
|
|
name string
|
|
expected int
|
|
getSelector func() selectors.Selector
|
|
}{
|
|
{
|
|
name: "Libraries",
|
|
getSelector: func() selectors.Selector {
|
|
sel := selectors.NewSharePointBackup(selSites)
|
|
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
|
return sel.Selector
|
|
},
|
|
},
|
|
{
|
|
name: "Lists",
|
|
expected: 0,
|
|
getSelector: func() selectors.Selector {
|
|
sel := selectors.NewSharePointBackup(selSites)
|
|
sel.Include(sel.Lists(selectors.Any()))
|
|
return sel.Selector
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
suite.Run(test.name, func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
sel := test.getSelector()
|
|
|
|
bpc := inject.BackupProducerConfig{
|
|
Options: control.DefaultOptions(),
|
|
ProtectedResource: sel,
|
|
Selector: sel,
|
|
}
|
|
|
|
collections, excludes, canUsePreviousBackup, err := sharepoint.ProduceBackupCollections(
|
|
ctx,
|
|
bpc,
|
|
suite.ac,
|
|
ctrl.credentials,
|
|
ctrl.UpdateStatus,
|
|
fault.New(true))
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
|
// Not expecting excludes as this isn't an incremental backup.
|
|
assert.True(t, excludes.Empty())
|
|
|
|
for range collections {
|
|
ctrl.incrementAwaitingMessages()
|
|
}
|
|
|
|
// we don't know an exact count of drives this will produce,
|
|
// but it should be more than one.
|
|
assert.Less(t, test.expected, len(collections))
|
|
|
|
for _, coll := range collections {
|
|
for object := range coll.Items(ctx, fault.New(true)) {
|
|
buf := &bytes.Buffer{}
|
|
_, err := buf.ReadFrom(object.ToReader())
|
|
assert.NoError(t, err, "reading item", clues.ToCore(err))
|
|
}
|
|
}
|
|
|
|
status := ctrl.Wait()
|
|
assert.NotZero(t, status.Successes)
|
|
t.Log(status.String())
|
|
})
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CreateSharePointCollection tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type SPCollectionIntgSuite struct {
|
|
tester.Suite
|
|
connector *Controller
|
|
user string
|
|
}
|
|
|
|
func TestSPCollectionIntgSuite(t *testing.T) {
|
|
suite.Run(t, &SPCollectionIntgSuite{
|
|
Suite: tester.NewIntegrationSuite(
|
|
t,
|
|
[][]string{tconfig.M365AcctCredEnvs}),
|
|
})
|
|
}
|
|
|
|
func (suite *SPCollectionIntgSuite) SetupSuite() {
|
|
ctx, flush := tester.NewContext(suite.T())
|
|
defer flush()
|
|
|
|
suite.connector = newController(ctx, suite.T(), path.SharePointService)
|
|
suite.user = tconfig.M365UserID(suite.T())
|
|
|
|
tester.LogTimeOfTest(suite.T())
|
|
}
|
|
|
|
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
var (
|
|
siteID = tconfig.M365SiteID(t)
|
|
ctrl = newController(ctx, t, path.SharePointService)
|
|
siteIDs = []string{siteID}
|
|
)
|
|
|
|
id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
sel := selectors.NewSharePointBackup(siteIDs)
|
|
sel.Include(sel.LibraryFolders([]string{"foo"}, selectors.PrefixMatch()))
|
|
|
|
sel.SetDiscreteOwnerIDName(id, name)
|
|
|
|
bpc := inject.BackupProducerConfig{
|
|
LastBackupVersion: version.NoBackup,
|
|
Options: control.DefaultOptions(),
|
|
ProtectedResource: inMock.NewProvider(id, name),
|
|
Selector: sel.Selector,
|
|
}
|
|
|
|
cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
|
ctx,
|
|
bpc,
|
|
fault.New(true))
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
|
require.Len(t, cols, 2) // 1 collection, 1 path prefix directory to ensure the root path exists.
|
|
// No excludes yet as this isn't an incremental backup.
|
|
assert.True(t, excludes.Empty())
|
|
|
|
t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String())
|
|
assert.Equal(
|
|
t,
|
|
path.SharePointMetadataService.String(),
|
|
cols[0].FullPath().Service().String())
|
|
|
|
t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String())
|
|
assert.Equal(
|
|
t,
|
|
path.SharePointService.String(),
|
|
cols[1].FullPath().Service().String())
|
|
}
|
|
|
|
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
var (
|
|
siteID = tconfig.M365SiteID(t)
|
|
ctrl = newController(ctx, t, path.SharePointService)
|
|
siteIDs = []string{siteID}
|
|
)
|
|
|
|
id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
sel := selectors.NewSharePointBackup(siteIDs)
|
|
sel.Include(sel.Lists(selectors.Any()))
|
|
|
|
sel.SetDiscreteOwnerIDName(id, name)
|
|
|
|
bpc := inject.BackupProducerConfig{
|
|
LastBackupVersion: version.NoBackup,
|
|
Options: control.DefaultOptions(),
|
|
ProtectedResource: inMock.NewProvider(id, name),
|
|
Selector: sel.Selector,
|
|
}
|
|
|
|
cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
|
ctx,
|
|
bpc,
|
|
fault.New(true))
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
|
assert.Less(t, 0, len(cols))
|
|
// No excludes yet as this isn't an incremental backup.
|
|
assert.True(t, excludes.Empty())
|
|
|
|
for _, collection := range cols {
|
|
t.Logf("Path: %s\n", collection.FullPath().String())
|
|
|
|
for item := range collection.Items(ctx, fault.New(true)) {
|
|
t.Log("File: " + item.ID())
|
|
|
|
_, err := io.ReadAll(item.ToReader())
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CreateGroupsCollection tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type GroupsCollectionIntgSuite struct {
|
|
tester.Suite
|
|
connector *Controller
|
|
tenantID string
|
|
user string
|
|
}
|
|
|
|
func TestGroupsCollectionIntgSuite(t *testing.T) {
|
|
suite.Run(t, &GroupsCollectionIntgSuite{
|
|
Suite: tester.NewIntegrationSuite(
|
|
t,
|
|
[][]string{tconfig.M365AcctCredEnvs}),
|
|
})
|
|
}
|
|
|
|
func (suite *GroupsCollectionIntgSuite) SetupSuite() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
suite.connector = newController(ctx, t, path.GroupsService)
|
|
suite.user = tconfig.M365UserID(t)
|
|
|
|
acct := tconfig.NewM365Account(t)
|
|
creds, err := acct.M365Config()
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
suite.tenantID = creds.AzureTenantID
|
|
|
|
tester.LogTimeOfTest(t)
|
|
}
|
|
|
|
func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
var (
|
|
groupID = tconfig.M365GroupID(t)
|
|
ctrl = newController(ctx, t, path.GroupsService)
|
|
groupIDs = []string{groupID}
|
|
)
|
|
|
|
id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, groupID, nil)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
sel := selectors.NewGroupsBackup(groupIDs)
|
|
sel.Include(sel.LibraryFolders([]string{"test"}, selectors.PrefixMatch()))
|
|
|
|
sel.SetDiscreteOwnerIDName(id, name)
|
|
|
|
bpc := inject.BackupProducerConfig{
|
|
LastBackupVersion: version.NoBackup,
|
|
Options: control.DefaultOptions(),
|
|
ProtectedResource: inMock.NewProvider(id, name),
|
|
Selector: sel.Selector,
|
|
}
|
|
|
|
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
|
ctx,
|
|
bpc,
|
|
fault.New(true))
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
|
// No excludes yet as this isn't an incremental backup.
|
|
assert.True(t, excludes.Empty())
|
|
|
|
// we don't know an exact count of drives this will produce,
|
|
// but it should be more than one.
|
|
assert.Greater(t, len(collections), 1)
|
|
|
|
p, err := path.BuildMetadata(
|
|
suite.tenantID,
|
|
groupID,
|
|
path.GroupsService,
|
|
path.LibrariesCategory,
|
|
false)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
p, err = p.Append(false, odConsts.SitesPathDir)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
foundSitesMetadata := false
|
|
|
|
for _, coll := range collections {
|
|
sitesMetadataCollection := coll.FullPath().String() == p.String()
|
|
|
|
for object := range coll.Items(ctx, fault.New(true)) {
|
|
if object.ID() == "previouspath" && sitesMetadataCollection {
|
|
foundSitesMetadata = true
|
|
}
|
|
|
|
buf := &bytes.Buffer{}
|
|
_, err := buf.ReadFrom(object.ToReader())
|
|
assert.NoError(t, err, "reading item", clues.ToCore(err))
|
|
}
|
|
}
|
|
|
|
assert.True(t, foundSitesMetadata, "missing sites metadata")
|
|
|
|
status := ctrl.Wait()
|
|
assert.NotZero(t, status.Successes)
|
|
t.Log(status.String())
|
|
}
|