ea6a57b2a5
<!-- PR description--> With #4497 changes, any counters that we log in graph middlewares ( e.g. `APICallTokensConsumed` or `ThrottledAPICalls`) will no longer show up in `BackupOperation.Results.Counts`. This is because graph adapters and backup operation will be initialized with different count busses. Since the backup result counts are captured from the operation bus, we will lose out on any graph metrics captured in middleware layers. This PR unifies the busses. Ill also add a regression check for graph metrics shortly --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [x] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #<issue> #### Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
1538 lines
41 KiB
Go
1538 lines
41 KiB
Go
package test_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/alcionai/clues"
|
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
|
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/stretchr/testify/suite"
|
|
"golang.org/x/exp/maps"
|
|
|
|
"github.com/alcionai/corso/src/internal/common/dttm"
|
|
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
|
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
|
"github.com/alcionai/corso/src/internal/events"
|
|
evmock "github.com/alcionai/corso/src/internal/events/mock"
|
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
|
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
|
|
exchTD "github.com/alcionai/corso/src/internal/m365/service/exchange/testdata"
|
|
"github.com/alcionai/corso/src/internal/tester"
|
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
|
"github.com/alcionai/corso/src/internal/version"
|
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
|
deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata"
|
|
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
|
"github.com/alcionai/corso/src/pkg/control"
|
|
ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
|
|
"github.com/alcionai/corso/src/pkg/count"
|
|
"github.com/alcionai/corso/src/pkg/fault"
|
|
"github.com/alcionai/corso/src/pkg/path"
|
|
"github.com/alcionai/corso/src/pkg/selectors"
|
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
|
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
|
|
)
|
|
|
|
type ExchangeBackupIntgSuite struct {
|
|
tester.Suite
|
|
its intgTesterSetup
|
|
}
|
|
|
|
func TestExchangeBackupIntgSuite(t *testing.T) {
|
|
suite.Run(t, &ExchangeBackupIntgSuite{
|
|
Suite: tester.NewIntegrationSuite(
|
|
t,
|
|
[][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}),
|
|
})
|
|
}
|
|
|
|
func (suite *ExchangeBackupIntgSuite) SetupSuite() {
|
|
suite.its = newIntegrationTesterSetup(suite.T())
|
|
}
|
|
|
|
// MetadataFileNames produces the category-specific set of filenames used to
|
|
// store graph metadata such as delta tokens and folderID->path references.
|
|
func MetadataFileNames(cat path.CategoryType) [][]string {
|
|
switch cat {
|
|
// TODO: should this include events?
|
|
case path.EmailCategory, path.ContactsCategory:
|
|
return [][]string{{metadata.DeltaURLsFileName}, {metadata.PreviousPathFileName}}
|
|
default:
|
|
return [][]string{{metadata.PreviousPathFileName}}
|
|
}
|
|
}
|
|
|
|
// TestBackup_Run ensures that Integration Testing works
|
|
// for the following scopes: Contacts, Events, and Mail
|
|
func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
|
|
tests := []struct {
|
|
name string
|
|
selector func() *selectors.ExchangeBackup
|
|
category path.CategoryType
|
|
metadataFiles [][]string
|
|
}{
|
|
{
|
|
name: "Mail",
|
|
selector: func() *selectors.ExchangeBackup {
|
|
sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
|
sel.DiscreteOwner = suite.its.user.ID
|
|
|
|
return sel
|
|
},
|
|
category: path.EmailCategory,
|
|
metadataFiles: MetadataFileNames(path.EmailCategory),
|
|
},
|
|
{
|
|
name: "Contacts",
|
|
selector: func() *selectors.ExchangeBackup {
|
|
sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()))
|
|
return sel
|
|
},
|
|
category: path.ContactsCategory,
|
|
metadataFiles: MetadataFileNames(path.ContactsCategory),
|
|
},
|
|
{
|
|
name: "Calendar Events",
|
|
selector: func() *selectors.ExchangeBackup {
|
|
sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()))
|
|
return sel
|
|
},
|
|
category: path.EventsCategory,
|
|
metadataFiles: MetadataFileNames(path.EventsCategory),
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
suite.Run(test.name, func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
var (
|
|
mb = evmock.NewBus()
|
|
counter = count.New()
|
|
sel = test.selector().Selector
|
|
opts = control.DefaultOptions()
|
|
whatSet = deeTD.CategoryFromRepoRef
|
|
)
|
|
|
|
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup, counter)
|
|
defer bod.close(t, ctx)
|
|
|
|
sel = bod.sel
|
|
|
|
userID := sel.ID()
|
|
|
|
m365, err := bod.acct.M365Config()
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
// run the tests
|
|
runAndCheckBackup(t, ctx, &bo, mb, false)
|
|
checkBackupIsInManifests(
|
|
t,
|
|
ctx,
|
|
bod.kw,
|
|
bod.sw,
|
|
&bo,
|
|
sel,
|
|
userID,
|
|
test.category)
|
|
checkMetadataFilesExist(
|
|
t,
|
|
ctx,
|
|
bo.Results.BackupID,
|
|
bod.kw,
|
|
bod.kms,
|
|
m365.AzureTenantID,
|
|
userID,
|
|
path.ExchangeService,
|
|
map[path.CategoryType][][]string{test.category: test.metadataFiles})
|
|
|
|
_, expectDeets := deeTD.GetDeetsInBackup(
|
|
t,
|
|
ctx,
|
|
bo.Results.BackupID,
|
|
bod.acct.ID(),
|
|
userID,
|
|
path.ExchangeService,
|
|
whatSet,
|
|
bod.kms,
|
|
bod.sss)
|
|
deeTD.CheckBackupDetails(
|
|
t,
|
|
ctx,
|
|
bo.Results.BackupID,
|
|
whatSet,
|
|
bod.kms,
|
|
bod.sss,
|
|
expectDeets,
|
|
false)
|
|
|
|
// Basic, happy path incremental test. No changes are dictated or expected.
|
|
// This only tests that an incremental backup is runnable at all, and that it
|
|
// produces fewer results than the last backup.
|
|
var (
|
|
incMB = evmock.NewBus()
|
|
incBO = newTestBackupOp(
|
|
t,
|
|
ctx,
|
|
bod,
|
|
incMB,
|
|
opts,
|
|
counter)
|
|
)
|
|
|
|
runAndCheckBackup(t, ctx, &incBO, incMB, true)
|
|
checkBackupIsInManifests(
|
|
t,
|
|
ctx,
|
|
bod.kw,
|
|
bod.sw,
|
|
&incBO,
|
|
sel,
|
|
userID,
|
|
test.category)
|
|
checkMetadataFilesExist(
|
|
t,
|
|
ctx,
|
|
incBO.Results.BackupID,
|
|
bod.kw,
|
|
bod.kms,
|
|
m365.AzureTenantID,
|
|
userID,
|
|
path.ExchangeService,
|
|
map[path.CategoryType][][]string{test.category: test.metadataFiles})
|
|
deeTD.CheckBackupDetails(
|
|
t,
|
|
ctx,
|
|
incBO.Results.BackupID,
|
|
whatSet,
|
|
bod.kms,
|
|
bod.sss,
|
|
expectDeets,
|
|
false)
|
|
|
|
// do some additional checks to ensure the incremental dealt with fewer items.
|
|
assert.Greater(t, bo.Results.ItemsWritten, incBO.Results.ItemsWritten, "incremental items written")
|
|
assert.Greater(t, bo.Results.ItemsRead, incBO.Results.ItemsRead, "incremental items read")
|
|
assert.Greater(t, bo.Results.BytesRead, incBO.Results.BytesRead, "incremental bytes read")
|
|
assert.Greater(t, bo.Results.BytesUploaded, incBO.Results.BytesUploaded, "incremental bytes uploaded")
|
|
assert.Equal(t, bo.Results.ResourceOwners, incBO.Results.ResourceOwners, "incremental backup resource owner")
|
|
assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure()))
|
|
assert.Empty(t, incBO.Errors.Recovered(), "count incremental recoverable/iteration errors")
|
|
assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events")
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchangeBasic_groups9VersionBump() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
var (
|
|
mb = evmock.NewBus()
|
|
counter = count.New()
|
|
sel = selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
opts = control.DefaultOptions()
|
|
ws = deeTD.DriveIDFromRepoRef
|
|
)
|
|
|
|
sel.Include(
|
|
sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()),
|
|
// sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()),
|
|
sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
|
|
|
bo, bod := prepNewTestBackupOp(
|
|
t,
|
|
ctx,
|
|
mb,
|
|
sel.Selector,
|
|
opts,
|
|
version.All8MigrateUserPNToID,
|
|
counter)
|
|
defer bod.close(t, ctx)
|
|
|
|
runAndCheckBackup(t, ctx, &bo, mb, false)
|
|
checkBackupIsInManifests(
|
|
t,
|
|
ctx,
|
|
bod.kw,
|
|
bod.sw,
|
|
&bo,
|
|
bod.sel,
|
|
bod.sel.ID(),
|
|
path.EmailCategory)
|
|
|
|
_, expectDeets := deeTD.GetDeetsInBackup(
|
|
t,
|
|
ctx,
|
|
bo.Results.BackupID,
|
|
bod.acct.ID(),
|
|
bod.sel.ID(),
|
|
path.ExchangeService,
|
|
ws,
|
|
bod.kms,
|
|
bod.sss)
|
|
deeTD.CheckBackupDetails(
|
|
t,
|
|
ctx,
|
|
bo.Results.BackupID,
|
|
ws,
|
|
bod.kms,
|
|
bod.sss,
|
|
expectDeets,
|
|
false)
|
|
|
|
mb = evmock.NewBus()
|
|
counter = count.New()
|
|
notForcedFull := newTestBackupOp(
|
|
t,
|
|
ctx,
|
|
bod,
|
|
mb,
|
|
opts,
|
|
counter)
|
|
notForcedFull.BackupVersion = version.Groups9Update
|
|
|
|
runAndCheckBackup(t, ctx, ¬ForcedFull, mb, false)
|
|
checkBackupIsInManifests(
|
|
t,
|
|
ctx,
|
|
bod.kw,
|
|
bod.sw,
|
|
¬ForcedFull,
|
|
bod.sel,
|
|
bod.sel.ID(),
|
|
path.EmailCategory)
|
|
|
|
_, expectDeets = deeTD.GetDeetsInBackup(
|
|
t,
|
|
ctx,
|
|
notForcedFull.Results.BackupID,
|
|
bod.acct.ID(),
|
|
bod.sel.ID(),
|
|
path.ExchangeService,
|
|
ws,
|
|
bod.kms,
|
|
bod.sss)
|
|
deeTD.CheckBackupDetails(
|
|
t,
|
|
ctx,
|
|
notForcedFull.Results.BackupID,
|
|
ws,
|
|
bod.kms,
|
|
bod.sss,
|
|
expectDeets,
|
|
false)
|
|
|
|
// The number of items backed up in the second backup should be less than the
|
|
// number of items in the original backup.
|
|
assert.Greater(
|
|
t,
|
|
bo.Results.Counts[string(count.PersistedNonCachedFiles)],
|
|
notForcedFull.Results.Counts[string(count.PersistedNonCachedFiles)],
|
|
"items written")
|
|
}
|
|
|
|
func (suite *ExchangeBackupIntgSuite) TestBackup_Run_incrementalExchange() {
|
|
testExchangeContinuousBackups(suite, control.Toggles{})
|
|
}
|
|
|
|
func (suite *ExchangeBackupIntgSuite) TestBackup_Run_incrementalNonDeltaExchange() {
|
|
testExchangeContinuousBackups(suite, control.Toggles{DisableDelta: true})
|
|
}
|
|
|
|
func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles control.Toggles) {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
tester.LogTimeOfTest(t)
|
|
|
|
var (
|
|
acct = tconfig.NewM365Account(t)
|
|
mb = evmock.NewBus()
|
|
counter = count.New()
|
|
now = dttm.Now()
|
|
service = path.ExchangeService
|
|
categories = map[path.CategoryType][][]string{
|
|
path.EmailCategory: MetadataFileNames(path.EmailCategory),
|
|
path.ContactsCategory: MetadataFileNames(path.ContactsCategory),
|
|
// path.EventsCategory: exchange.MetadataFileNames(path.EventsCategory),
|
|
}
|
|
container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now)
|
|
container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now)
|
|
container3 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 3, now)
|
|
containerRename = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 4, now)
|
|
|
|
// container3 and containerRename don't exist yet. Those will get created
|
|
// later on during the tests. Putting their identifiers into the selector
|
|
// at this point is harmless.
|
|
containers = []string{container1, container2, container3, containerRename}
|
|
sel = selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
whatSet = deeTD.CategoryFromRepoRef
|
|
opts = control.DefaultOptions()
|
|
)
|
|
|
|
opts.ToggleFeatures = toggles
|
|
ctrl, sels := ControllerWithSelector(t, ctx, acct, sel.Selector, nil, nil, counter)
|
|
sel.DiscreteOwner = sels.ID()
|
|
sel.DiscreteOwnerName = sels.Name()
|
|
|
|
uidn := inMock.NewProvider(sels.ID(), sels.Name())
|
|
|
|
sel.Include(
|
|
sel.ContactFolders(containers, selectors.PrefixMatch()),
|
|
// sel.EventCalendars(containers, selectors.PrefixMatch()),
|
|
sel.MailFolders(containers, selectors.PrefixMatch()))
|
|
|
|
creds, err := acct.M365Config()
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
ac, err := api.NewClient(
|
|
creds,
|
|
control.DefaultOptions(),
|
|
count.New())
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
// generate 3 new folders with two items each.
|
|
// Only the first two folders will be part of the initial backup and
|
|
// incrementals. The third folder will be introduced partway through
|
|
// the changes.
|
|
// This should be enough to cover most delta actions, since moving one
|
|
// container into another generates a delta for both addition and deletion.
|
|
type contDeets struct {
|
|
containerID string
|
|
locRef string
|
|
itemRefs []string // cached for populating expected deets, otherwise not used
|
|
}
|
|
|
|
mailDBF := func(id, timeStamp, subject, body string) []byte {
|
|
return exchMock.MessageWith(
|
|
suite.its.user.ID, suite.its.user.ID, suite.its.user.ID,
|
|
subject, body, body,
|
|
now, now, now, now)
|
|
}
|
|
|
|
contactDBF := func(id, timeStamp, subject, body string) []byte {
|
|
given, mid, sur := id[:8], id[9:13], id[len(id)-12:]
|
|
|
|
return exchMock.ContactBytesWith(
|
|
given+" "+sur,
|
|
sur+", "+given,
|
|
given, mid, sur,
|
|
"123-456-7890")
|
|
}
|
|
|
|
eventDBF := func(id, timeStamp, subject, body string) []byte {
|
|
return exchMock.EventWith(
|
|
suite.its.user.ID, subject, body, body,
|
|
exchMock.NoOriginalStartDate, now, now,
|
|
exchMock.NoRecurrence, exchMock.NoAttendees,
|
|
exchMock.NoAttachments, exchMock.NoCancelledOccurrences,
|
|
exchMock.NoExceptionOccurrences)
|
|
}
|
|
|
|
// test data set
|
|
dataset := map[path.CategoryType]struct {
|
|
dbf dataBuilderFunc
|
|
dests map[string]contDeets
|
|
}{
|
|
path.EmailCategory: {
|
|
dbf: mailDBF,
|
|
dests: map[string]contDeets{
|
|
container1: {},
|
|
container2: {},
|
|
},
|
|
},
|
|
path.ContactsCategory: {
|
|
dbf: contactDBF,
|
|
dests: map[string]contDeets{
|
|
container1: {},
|
|
container2: {},
|
|
},
|
|
},
|
|
// path.EventsCategory: {
|
|
// dbf: eventDBF,
|
|
// dests: map[string]contDeets{
|
|
// container1: {},
|
|
// container2: {},
|
|
// },
|
|
// },
|
|
}
|
|
|
|
rrPfx, err := path.BuildPrefix(acct.ID(), uidn.ID(), service, path.EmailCategory)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
// strip the category from the prefix; we primarily want the tenant and resource owner.
|
|
expectDeets := deeTD.NewInDeets(rrPfx.ToBuilder().Dir().String())
|
|
|
|
mustGetExpectedContainerItems := func(
|
|
t *testing.T,
|
|
category path.CategoryType,
|
|
cr graph.ContainerResolver,
|
|
destName string,
|
|
) {
|
|
locRef := path.Builder{}.Append(destName)
|
|
|
|
if category == path.EmailCategory {
|
|
locRef = locRef.Append(destName)
|
|
}
|
|
|
|
containerID, ok := cr.LocationInCache(locRef.String())
|
|
require.True(t, ok, "dir %s found in %s cache", locRef.String(), category)
|
|
|
|
var (
|
|
err error
|
|
items map[string]time.Time
|
|
)
|
|
|
|
switch category {
|
|
case path.EmailCategory:
|
|
items, _, _, _, err = ac.Mail().GetAddedAndRemovedItemIDs(
|
|
ctx,
|
|
uidn.ID(),
|
|
containerID,
|
|
"",
|
|
toggles.ExchangeImmutableIDs,
|
|
true)
|
|
|
|
case path.EventsCategory:
|
|
items, _, _, _, err = ac.Events().GetAddedAndRemovedItemIDs(
|
|
ctx,
|
|
uidn.ID(),
|
|
containerID,
|
|
"",
|
|
toggles.ExchangeImmutableIDs,
|
|
true)
|
|
|
|
case path.ContactsCategory:
|
|
items, _, _, _, err = ac.Contacts().GetAddedAndRemovedItemIDs(
|
|
ctx,
|
|
uidn.ID(),
|
|
containerID,
|
|
"",
|
|
toggles.ExchangeImmutableIDs,
|
|
true)
|
|
}
|
|
|
|
require.NoError(
|
|
t,
|
|
err,
|
|
"getting items for category %s, container %s",
|
|
category,
|
|
locRef.String())
|
|
|
|
dest := dataset[category].dests[destName]
|
|
dest.locRef = locRef.String()
|
|
dest.containerID = containerID
|
|
dest.itemRefs = maps.Keys(items)
|
|
dataset[category].dests[destName] = dest
|
|
|
|
// Add the directory and all its ancestors to the cache so we can compare
|
|
// folders.
|
|
for len(locRef.Elements()) > 0 {
|
|
expectDeets.AddLocation(category.String(), locRef.String())
|
|
locRef = locRef.Dir()
|
|
}
|
|
|
|
for _, i := range dataset[category].dests[destName].itemRefs {
|
|
expectDeets.AddItem(category.String(), dest.locRef, i)
|
|
}
|
|
}
|
|
|
|
// populate initial test data
|
|
for category, gen := range dataset {
|
|
for destName := range gen.dests {
|
|
generateContainerOfItems(
|
|
t,
|
|
ctx,
|
|
ctrl,
|
|
service,
|
|
category,
|
|
selectors.NewExchangeRestore([]string{uidn.ID()}).Selector,
|
|
creds.AzureTenantID,
|
|
uidn.ID(),
|
|
"",
|
|
"",
|
|
destName,
|
|
2,
|
|
version.Backup,
|
|
gen.dbf)
|
|
}
|
|
|
|
cr := exchTD.PopulateContainerCache(
|
|
t,
|
|
ctx,
|
|
ac,
|
|
category,
|
|
uidn.ID(),
|
|
fault.New(true))
|
|
|
|
for destName := range gen.dests {
|
|
mustGetExpectedContainerItems(t, category, cr, destName)
|
|
}
|
|
}
|
|
|
|
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup, counter)
|
|
defer bod.close(t, ctx)
|
|
|
|
// run the initial backup
|
|
runAndCheckBackup(t, ctx, &bo, mb, false)
|
|
|
|
// precheck to ensure the expectedDeets are correct.
|
|
// if we fail here, the expectedDeets were populated incorrectly.
|
|
deeTD.CheckBackupDetails(
|
|
t,
|
|
ctx,
|
|
bo.Results.BackupID,
|
|
whatSet,
|
|
bod.kms,
|
|
bod.sss,
|
|
expectDeets,
|
|
true)
|
|
|
|
// Although established as a table, these tests are not isolated from each other.
|
|
// Assume that every test's side effects cascade to all following test cases.
|
|
// The changes are split across the table so that we can monitor the deltas
|
|
// in isolation, rather than debugging one change from the rest of a series.
|
|
table := []struct {
|
|
name string
|
|
// performs the incremental update required for the test.
|
|
//revive:disable-next-line:context-as-argument
|
|
updateUserData func(t *testing.T, ctx context.Context)
|
|
deltaItemsRead int
|
|
deltaItemsWritten int
|
|
nonDeltaItemsRead int
|
|
nonDeltaItemsWritten int
|
|
nonMetaItemsWritten int
|
|
}{
|
|
{
|
|
name: "clean, no changes",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {},
|
|
deltaItemsRead: 0,
|
|
deltaItemsWritten: 0,
|
|
nonDeltaItemsRead: 8,
|
|
nonDeltaItemsWritten: 0, // unchanged items are not counted towards write
|
|
nonMetaItemsWritten: 4,
|
|
},
|
|
{
|
|
name: "move an email folder to a subfolder",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
cat := path.EmailCategory
|
|
|
|
// contacts and events cannot be sufoldered; this is an email-only change
|
|
from := dataset[cat].dests[container2]
|
|
to := dataset[cat].dests[container1]
|
|
|
|
body := users.NewItemMailFoldersItemMovePostRequestBody()
|
|
body.SetDestinationId(ptr.To(to.containerID))
|
|
|
|
err := ac.Mail().MoveContainer(ctx, uidn.ID(), from.containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
newLoc := expectDeets.MoveLocation(cat.String(), from.locRef, to.locRef)
|
|
|
|
// Remove ancestor folders of moved directory since they'll no longer
|
|
// appear in details since we're not backing up items in them.
|
|
pb, err := path.Builder{}.SplitUnescapeAppend(from.locRef)
|
|
require.NoError(t, err, "getting Builder for location: %s", clues.ToCore(err))
|
|
|
|
pb = pb.Dir()
|
|
|
|
for len(pb.Elements()) > 0 {
|
|
expectDeets.RemoveLocation(cat.String(), pb.String())
|
|
pb = pb.Dir()
|
|
}
|
|
|
|
// Update cache with new location of container.
|
|
from.locRef = newLoc
|
|
dataset[cat].dests[container2] = from
|
|
},
|
|
deltaItemsRead: 0, // zero because we don't count container reads
|
|
deltaItemsWritten: 2,
|
|
nonDeltaItemsRead: 8,
|
|
nonDeltaItemsWritten: 2,
|
|
nonMetaItemsWritten: 6,
|
|
},
|
|
{
|
|
name: "delete a folder",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
for category, d := range dataset {
|
|
containerID := d.dests[container2].containerID
|
|
|
|
switch category {
|
|
case path.EmailCategory:
|
|
err := ac.Mail().DeleteContainer(ctx, uidn.ID(), containerID)
|
|
require.NoError(t, err, "deleting an email folder", clues.ToCore(err))
|
|
case path.ContactsCategory:
|
|
err := ac.Contacts().DeleteContainer(ctx, uidn.ID(), containerID)
|
|
require.NoError(t, err, "deleting a contacts folder", clues.ToCore(err))
|
|
case path.EventsCategory:
|
|
err := ac.Events().DeleteContainer(ctx, uidn.ID(), containerID)
|
|
require.NoError(t, err, "deleting a calendar", clues.ToCore(err))
|
|
}
|
|
|
|
expectDeets.RemoveLocation(category.String(), d.dests[container2].locRef)
|
|
}
|
|
},
|
|
deltaItemsRead: 0,
|
|
deltaItemsWritten: 0, // deletions are not counted as "writes"
|
|
nonDeltaItemsRead: 4,
|
|
nonDeltaItemsWritten: 0,
|
|
nonMetaItemsWritten: 4,
|
|
},
|
|
{
|
|
name: "add a new folder",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
for category, gen := range dataset {
|
|
generateContainerOfItems(
|
|
t,
|
|
ctx,
|
|
ctrl,
|
|
service,
|
|
category,
|
|
selectors.NewExchangeRestore([]string{uidn.ID()}).Selector,
|
|
creds.AzureTenantID, suite.its.user.ID, "", "", container3,
|
|
2,
|
|
version.Backup,
|
|
gen.dbf)
|
|
|
|
cr := exchTD.PopulateContainerCache(t, ctx, ac, category, uidn.ID(), fault.New(true))
|
|
mustGetExpectedContainerItems(t, category, cr, container3)
|
|
}
|
|
},
|
|
deltaItemsRead: 4,
|
|
deltaItemsWritten: 4,
|
|
nonDeltaItemsRead: 8,
|
|
nonDeltaItemsWritten: 4,
|
|
nonMetaItemsWritten: 8,
|
|
},
|
|
{
|
|
name: "rename a folder",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
for category, d := range dataset {
|
|
containerID := d.dests[container3].containerID
|
|
newLoc := containerRename
|
|
|
|
if category == path.EmailCategory {
|
|
newLoc = path.Builder{}.Append(container3, containerRename).String()
|
|
}
|
|
|
|
d.dests[containerRename] = contDeets{
|
|
containerID: containerID,
|
|
locRef: newLoc,
|
|
}
|
|
|
|
expectDeets.RenameLocation(
|
|
category.String(),
|
|
d.dests[container3].locRef,
|
|
newLoc)
|
|
|
|
switch category {
|
|
case path.EmailCategory:
|
|
body := models.NewMailFolder()
|
|
body.SetDisplayName(ptr.To(containerRename))
|
|
|
|
err := ac.Mail().PatchFolder(ctx, uidn.ID(), containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
case path.ContactsCategory:
|
|
body := models.NewContactFolder()
|
|
body.SetDisplayName(ptr.To(containerRename))
|
|
|
|
err = ac.Contacts().PatchFolder(ctx, uidn.ID(), containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
case path.EventsCategory:
|
|
body := models.NewCalendar()
|
|
body.SetName(ptr.To(containerRename))
|
|
|
|
err = ac.Events().PatchCalendar(ctx, uidn.ID(), containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
}
|
|
}
|
|
},
|
|
deltaItemsRead: 0, // containers are not counted as reads
|
|
// Renaming a folder doesn't cause kopia changes as the folder ID doesn't
|
|
// change.
|
|
deltaItemsWritten: 0,
|
|
nonDeltaItemsRead: 8,
|
|
nonDeltaItemsWritten: 0,
|
|
nonMetaItemsWritten: 4,
|
|
},
|
|
{
|
|
name: "add a new item",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
for category, d := range dataset {
|
|
containerID := d.dests[container1].containerID
|
|
|
|
switch category {
|
|
case path.EmailCategory:
|
|
_, itemData := generateItemData(t, category, uidn.ID(), mailDBF)
|
|
body, err := api.BytesToMessageable(itemData)
|
|
require.NoErrorf(t, err, "transforming mail bytes to messageable: %+v", clues.ToCore(err))
|
|
|
|
itm, err := ac.Mail().PostItem(ctx, uidn.ID(), containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
expectDeets.AddItem(
|
|
category.String(),
|
|
d.dests[container1].locRef,
|
|
ptr.Val(itm.GetId()))
|
|
|
|
case path.ContactsCategory:
|
|
_, itemData := generateItemData(t, category, uidn.ID(), contactDBF)
|
|
body, err := api.BytesToContactable(itemData)
|
|
require.NoErrorf(t, err, "transforming contact bytes to contactable: %+v", clues.ToCore(err))
|
|
|
|
itm, err := ac.Contacts().PostItem(ctx, uidn.ID(), containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
expectDeets.AddItem(
|
|
category.String(),
|
|
d.dests[container1].locRef,
|
|
ptr.Val(itm.GetId()))
|
|
|
|
case path.EventsCategory:
|
|
_, itemData := generateItemData(t, category, uidn.ID(), eventDBF)
|
|
body, err := api.BytesToEventable(itemData)
|
|
require.NoErrorf(t, err, "transforming event bytes to eventable: %+v", clues.ToCore(err))
|
|
|
|
itm, err := ac.Events().PostItem(ctx, uidn.ID(), containerID, body)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
expectDeets.AddItem(
|
|
category.String(),
|
|
d.dests[container1].locRef,
|
|
ptr.Val(itm.GetId()))
|
|
}
|
|
}
|
|
},
|
|
deltaItemsRead: 2,
|
|
deltaItemsWritten: 2,
|
|
nonDeltaItemsRead: 10,
|
|
nonDeltaItemsWritten: 2,
|
|
nonMetaItemsWritten: 6,
|
|
},
|
|
{
|
|
// Events and contacts have no Graph API call to move something between
|
|
// containers. The calendars web UI does support moving events between
|
|
// calendars though.
|
|
name: "boomerang an email",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
containerInfo := dataset[path.EmailCategory].dests[container1]
|
|
tempContainerID := dataset[path.EmailCategory].dests[container3].containerID
|
|
|
|
ids := dataset[path.EmailCategory].dests[container1].itemRefs
|
|
require.NotEmpty(t, ids, "message ids in folder")
|
|
|
|
oldID := ids[0]
|
|
|
|
newID, err := ac.Mail().MoveItem(
|
|
ctx,
|
|
uidn.ID(),
|
|
containerInfo.containerID,
|
|
tempContainerID,
|
|
oldID)
|
|
require.NoError(t, err, "moving to temp folder: %s", clues.ToCore(err))
|
|
|
|
newID, err = ac.Mail().MoveItem(
|
|
ctx,
|
|
uidn.ID(),
|
|
tempContainerID,
|
|
containerInfo.containerID,
|
|
newID)
|
|
require.NoError(t, err, "moving back to original folder: %s", clues.ToCore(err))
|
|
|
|
expectDeets.RemoveItem(
|
|
path.EmailCategory.String(),
|
|
containerInfo.locRef,
|
|
oldID)
|
|
expectDeets.AddItem(
|
|
path.EmailCategory.String(),
|
|
containerInfo.locRef,
|
|
newID)
|
|
|
|
// Will cause a different item to be deleted next.
|
|
containerInfo.itemRefs = append(containerInfo.itemRefs[1:], newID)
|
|
dataset[path.EmailCategory].dests[container1] = containerInfo
|
|
},
|
|
// TODO(ashmrtn): Below values need updated when we start checking them
|
|
// again. Unclear what items would be considered the same as I'm not sure
|
|
// about all the properties that change with a move.
|
|
deltaItemsRead: 2,
|
|
deltaItemsWritten: 2,
|
|
nonDeltaItemsRead: 10,
|
|
nonDeltaItemsWritten: 2,
|
|
nonMetaItemsWritten: 6,
|
|
},
|
|
{
|
|
name: "delete an existing item",
|
|
updateUserData: func(t *testing.T, ctx context.Context) {
|
|
for category, d := range dataset {
|
|
containerInfo := d.dests[container1]
|
|
require.NotEmpty(t, containerInfo.itemRefs)
|
|
|
|
id := containerInfo.itemRefs[0]
|
|
|
|
switch category {
|
|
case path.EmailCategory:
|
|
err = ac.Mail().DeleteItem(ctx, uidn.ID(), id)
|
|
|
|
case path.ContactsCategory:
|
|
err = ac.Contacts().DeleteItem(ctx, uidn.ID(), id)
|
|
|
|
case path.EventsCategory:
|
|
err = ac.Events().DeleteItem(ctx, uidn.ID(), id)
|
|
}
|
|
|
|
require.NoError(
|
|
t,
|
|
err,
|
|
"deleting %s item: %s",
|
|
category.String(),
|
|
clues.ToCore(err))
|
|
expectDeets.RemoveItem(category.String(), containerInfo.locRef, id)
|
|
}
|
|
},
|
|
deltaItemsRead: 2,
|
|
deltaItemsWritten: 0, // deletes are not counted as "writes"
|
|
nonDeltaItemsRead: 8,
|
|
nonDeltaItemsWritten: 0,
|
|
nonMetaItemsWritten: 4,
|
|
},
|
|
}
|
|
|
|
for _, test := range table {
|
|
suite.Run(test.name, func() {
|
|
var (
|
|
t = suite.T()
|
|
incMB = evmock.NewBus()
|
|
counter = count.New()
|
|
atid = creds.AzureTenantID
|
|
)
|
|
|
|
ctx, flush := tester.WithContext(t, ctx)
|
|
defer flush()
|
|
|
|
incBO := newTestBackupOp(t, ctx, bod, incMB, opts, counter)
|
|
|
|
suite.Run("PreTestSetup", func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.WithContext(t, ctx)
|
|
defer flush()
|
|
|
|
test.updateUserData(t, ctx)
|
|
})
|
|
|
|
err := incBO.Run(ctx)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
bupID := incBO.Results.BackupID
|
|
|
|
checkBackupIsInManifests(
|
|
t,
|
|
ctx,
|
|
bod.kw,
|
|
bod.sw,
|
|
&incBO,
|
|
sels,
|
|
uidn.ID(),
|
|
maps.Keys(categories)...)
|
|
checkMetadataFilesExist(
|
|
t,
|
|
ctx,
|
|
bupID,
|
|
bod.kw,
|
|
bod.kms,
|
|
atid,
|
|
uidn.ID(),
|
|
service,
|
|
categories)
|
|
deeTD.CheckBackupDetails(
|
|
t,
|
|
ctx,
|
|
bupID,
|
|
whatSet,
|
|
bod.kms,
|
|
bod.sss,
|
|
expectDeets,
|
|
true)
|
|
|
|
// FIXME: commented tests are flaky due to delta calls retaining data that is
|
|
// out of scope of the test data.
|
|
// we need to find a better way to make isolated assertions here.
|
|
// The addition of the deeTD package gives us enough coverage to comment
|
|
// out the tests for now and look to their improvemeng later.
|
|
|
|
// do some additional checks to ensure the incremental dealt with fewer items.
|
|
// +4 on read/writes to account for metadata: 1 delta and 1 path for each type.
|
|
// if !toggles.DisableDelta {
|
|
// assert.Equal(t, test.deltaItemsRead+4, incBO.Results.ItemsRead, "incremental items read")
|
|
// assert.Equal(t, test.deltaItemsWritten+4, incBO.Results.ItemsWritten, "incremental items written")
|
|
// } else {
|
|
// assert.Equal(t, test.nonDeltaItemsRead+4, incBO.Results.ItemsRead, "non delta items read")
|
|
// assert.Equal(t, test.nonDeltaItemsWritten+4, incBO.Results.ItemsWritten, "non delta items written")
|
|
// }
|
|
// assert.Equal(t, test.nonMetaItemsWritten, incBO.Results.ItemsWritten, "non meta incremental items write")
|
|
assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(incBO.Errors.Failure()))
|
|
assert.Empty(t, incBO.Errors.Recovered(), "incremental recoverable/iteration errors")
|
|
assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events")
|
|
})
|
|
}
|
|
}
|
|
|
|
type ExchangeRestoreNightlyIntgSuite struct {
|
|
tester.Suite
|
|
its intgTesterSetup
|
|
}
|
|
|
|
func TestExchangeRestoreIntgSuite(t *testing.T) {
|
|
suite.Run(t, &ExchangeRestoreNightlyIntgSuite{
|
|
Suite: tester.NewNightlySuite(
|
|
t,
|
|
[][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}),
|
|
})
|
|
}
|
|
|
|
func (suite *ExchangeRestoreNightlyIntgSuite) SetupSuite() {
|
|
suite.its = newIntegrationTesterSetup(suite.T())
|
|
}
|
|
|
|
type clientItemPager interface {
|
|
GetItemsInContainerByCollisionKeyer[string]
|
|
GetItemIDsInContainer(
|
|
ctx context.Context,
|
|
userID, containerID string,
|
|
) (map[string]struct{}, error)
|
|
GetContainerByName(
|
|
ctx context.Context,
|
|
userID, parentContainerID, containerName string,
|
|
) (graph.Container, error)
|
|
GetItemsInContainerByCollisionKey(
|
|
ctx context.Context,
|
|
userID, containerID string,
|
|
) (map[string]string, error)
|
|
CreateContainer(
|
|
ctx context.Context,
|
|
userID, parentContainerID, containerName string,
|
|
) (graph.Container, error)
|
|
}
|
|
|
|
func (suite *ExchangeRestoreNightlyIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
// a backup is required to run restores
|
|
|
|
baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
baseSel.Include(
|
|
// events cannot be run, for the same reason as incremental backups: the user needs
|
|
// to have their account recycled.
|
|
// base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()),
|
|
baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()),
|
|
baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
|
|
|
baseSel.DiscreteOwner = suite.its.user.ID
|
|
|
|
var (
|
|
mb = evmock.NewBus()
|
|
counter = count.New()
|
|
opts = control.DefaultOptions()
|
|
)
|
|
|
|
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup, counter)
|
|
defer bod.close(t, ctx)
|
|
|
|
runAndCheckBackup(t, ctx, &bo, mb, false)
|
|
|
|
rsel, err := baseSel.ToExchangeRestore()
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
var (
|
|
restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore")
|
|
sel = rsel.Selector
|
|
userID = sel.ID()
|
|
countItemsInRestore int
|
|
|
|
itemIDs = map[path.CategoryType]map[string]struct{}{}
|
|
collisionKeys = map[path.CategoryType]map[string]string{}
|
|
containerIDs = map[path.CategoryType]string{}
|
|
parentContainerIDs = map[path.CategoryType]string{
|
|
path.EmailCategory: api.MsgFolderRoot,
|
|
}
|
|
parentContainerNames = map[path.CategoryType][]string{
|
|
path.EmailCategory: {api.MailInbox},
|
|
path.ContactsCategory: {},
|
|
path.EventsCategory: {},
|
|
}
|
|
|
|
testCategories = map[path.CategoryType]clientItemPager{
|
|
path.ContactsCategory: suite.its.ac.Contacts(),
|
|
path.EmailCategory: suite.its.ac.Mail(),
|
|
// path.EventsCategory: suite.its.ac.Events(),
|
|
}
|
|
)
|
|
|
|
// initial restore
|
|
|
|
suite.Run("baseline", func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
mb := evmock.NewBus()
|
|
ctr1 := count.New()
|
|
|
|
restoreCfg.OnCollision = control.Copy
|
|
|
|
ro, _ := prepNewTestRestoreOp(
|
|
t,
|
|
ctx,
|
|
bod.st,
|
|
bo.Results.BackupID,
|
|
mb,
|
|
ctr1,
|
|
sel,
|
|
opts,
|
|
restoreCfg)
|
|
|
|
runAndCheckRestore(t, ctx, &ro, mb, false)
|
|
|
|
// get all files in folder, use these as the base
|
|
// set of files to compare against.
|
|
|
|
for cat, ac := range testCategories {
|
|
suite.Run(cat.String(), func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
containers := append([]string{restoreCfg.Location}, parentContainerNames[cat]...)
|
|
|
|
itemIDs[cat], collisionKeys[cat], containerIDs[cat] = getCollKeysAndItemIDs(
|
|
t,
|
|
ctx,
|
|
ac,
|
|
userID,
|
|
parentContainerIDs[cat],
|
|
containers...)
|
|
|
|
countItemsInRestore += len(collisionKeys[cat])
|
|
})
|
|
}
|
|
|
|
checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore)
|
|
})
|
|
|
|
// skip restore
|
|
|
|
suite.Run("skip collisions", func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
mb := evmock.NewBus()
|
|
ctr2 := count.New()
|
|
|
|
restoreCfg.OnCollision = control.Skip
|
|
|
|
ro, _ := prepNewTestRestoreOp(
|
|
t,
|
|
ctx,
|
|
bod.st,
|
|
bo.Results.BackupID,
|
|
mb,
|
|
ctr2,
|
|
sel,
|
|
opts,
|
|
restoreCfg)
|
|
|
|
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
|
|
|
|
assert.Zero(
|
|
t,
|
|
len(deets.Entries),
|
|
"no items should have been restored")
|
|
|
|
checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0)
|
|
|
|
result := map[string]string{}
|
|
|
|
for cat, ac := range testCategories {
|
|
suite.Run(cat.String(), func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
m := filterCollisionKeyResults(
|
|
t,
|
|
ctx,
|
|
userID,
|
|
containerIDs[cat],
|
|
GetItemsInContainerByCollisionKeyer[string](ac),
|
|
collisionKeys[cat])
|
|
maps.Copy(result, m)
|
|
|
|
currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat])
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
assert.Equal(t, itemIDs[cat], currentIDs, "ids are equal")
|
|
})
|
|
}
|
|
|
|
assert.Len(t, result, 0, "no new items should get added")
|
|
})
|
|
|
|
// replace restore
|
|
|
|
suite.Run("replace collisions", func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
mb := evmock.NewBus()
|
|
ctr3 := count.New()
|
|
|
|
restoreCfg.OnCollision = control.Replace
|
|
|
|
ro, _ := prepNewTestRestoreOp(
|
|
t,
|
|
ctx,
|
|
bod.st,
|
|
bo.Results.BackupID,
|
|
mb,
|
|
ctr3,
|
|
sel,
|
|
opts,
|
|
restoreCfg)
|
|
|
|
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
|
|
filtEnts := []details.Entry{}
|
|
|
|
for _, e := range deets.Entries {
|
|
if e.Folder == nil {
|
|
filtEnts = append(filtEnts, e)
|
|
}
|
|
}
|
|
|
|
assert.Len(t, filtEnts, countItemsInRestore, "every item should have been replaced")
|
|
|
|
checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0)
|
|
|
|
result := map[string]string{}
|
|
|
|
for cat, ac := range testCategories {
|
|
suite.Run(cat.String(), func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
m := filterCollisionKeyResults(
|
|
t,
|
|
ctx,
|
|
userID,
|
|
containerIDs[cat],
|
|
GetItemsInContainerByCollisionKeyer[string](ac),
|
|
collisionKeys[cat])
|
|
maps.Copy(result, m)
|
|
|
|
currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat])
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
assert.Equal(t, len(itemIDs[cat]), len(currentIDs), "count of ids are equal")
|
|
for orig := range itemIDs[cat] {
|
|
assert.NotContains(t, currentIDs, orig, "original item should not exist after replacement")
|
|
}
|
|
|
|
itemIDs[cat] = currentIDs
|
|
})
|
|
}
|
|
|
|
assert.Len(t, result, 0, "all items should have been replaced")
|
|
})
|
|
|
|
// copy restore
|
|
|
|
suite.Run("copy collisions", func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
mb := evmock.NewBus()
|
|
ctr4 := count.New()
|
|
|
|
restoreCfg.OnCollision = control.Copy
|
|
|
|
ro, _ := prepNewTestRestoreOp(
|
|
t,
|
|
ctx,
|
|
bod.st,
|
|
bo.Results.BackupID,
|
|
mb,
|
|
ctr4,
|
|
sel,
|
|
opts,
|
|
restoreCfg)
|
|
|
|
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
|
|
filtEnts := []details.Entry{}
|
|
|
|
for _, e := range deets.Entries {
|
|
if e.Folder == nil {
|
|
filtEnts = append(filtEnts, e)
|
|
}
|
|
}
|
|
|
|
assert.Len(t, filtEnts, countItemsInRestore, "every item should have been copied")
|
|
|
|
checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore)
|
|
|
|
result := map[string]string{}
|
|
|
|
for cat, ac := range testCategories {
|
|
suite.Run(cat.String(), func() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
m := filterCollisionKeyResults(
|
|
t,
|
|
ctx,
|
|
userID,
|
|
containerIDs[cat],
|
|
GetItemsInContainerByCollisionKeyer[string](ac),
|
|
collisionKeys[cat])
|
|
maps.Copy(result, m)
|
|
|
|
currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat])
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
assert.Equal(t, 2*len(itemIDs[cat]), len(currentIDs), "count of ids should be double from before")
|
|
assert.Subset(t, maps.Keys(currentIDs), maps.Keys(itemIDs[cat]), "original item should exist after copy")
|
|
})
|
|
}
|
|
|
|
// TODO: we have the option of modifying copy creations in exchange
|
|
// so that the results don't collide. But we haven't made that
|
|
// decision yet.
|
|
assert.Len(t, result, 0, "no items should have been added as copies")
|
|
})
|
|
}
|
|
|
|
func (suite *ExchangeRestoreNightlyIntgSuite) TestRestore_Run_exchangeAlternateProtectedResource() {
|
|
t := suite.T()
|
|
|
|
ctx, flush := tester.NewContext(t)
|
|
defer flush()
|
|
|
|
// a backup is required to run restores
|
|
|
|
baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
|
baseSel.Include(
|
|
// events cannot be run, for the same reason as incremental backups: the user needs
|
|
// to have their account recycled.
|
|
// base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()),
|
|
baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()),
|
|
baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
|
|
|
baseSel.DiscreteOwner = suite.its.user.ID
|
|
|
|
var (
|
|
mb = evmock.NewBus()
|
|
counter = count.New()
|
|
opts = control.DefaultOptions()
|
|
)
|
|
|
|
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup, counter)
|
|
defer bod.close(t, ctx)
|
|
|
|
runAndCheckBackup(t, ctx, &bo, mb, false)
|
|
|
|
rsel, err := baseSel.ToExchangeRestore()
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
var (
|
|
restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_restore_to_user")
|
|
sel = rsel.Selector
|
|
userID = suite.its.user.ID
|
|
secondaryUserID = suite.its.secondaryUser.ID
|
|
uid = userID
|
|
acCont = suite.its.ac.Contacts()
|
|
acMail = suite.its.ac.Mail()
|
|
// acEvts = suite.its.ac.Events()
|
|
firstCtr = count.New()
|
|
)
|
|
|
|
restoreCfg.OnCollision = control.Copy
|
|
mb = evmock.NewBus()
|
|
|
|
// first restore to the current user
|
|
|
|
ro1, _ := prepNewTestRestoreOp(
|
|
t,
|
|
ctx,
|
|
bod.st,
|
|
bo.Results.BackupID,
|
|
mb,
|
|
firstCtr,
|
|
sel,
|
|
opts,
|
|
restoreCfg)
|
|
|
|
runAndCheckRestore(t, ctx, &ro1, mb, false)
|
|
|
|
// get all files in folder, use these as the base
|
|
// set of files to compare against.
|
|
|
|
var (
|
|
userItemIDs = map[path.CategoryType]map[string]struct{}{}
|
|
userCollisionKeys = map[path.CategoryType]map[string]string{}
|
|
)
|
|
|
|
// --- contacts
|
|
cat := path.ContactsCategory
|
|
userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs(
|
|
t,
|
|
ctx,
|
|
acCont,
|
|
uid,
|
|
"",
|
|
restoreCfg.Location)
|
|
|
|
// --- events
|
|
// cat = path.EventsCategory
|
|
// userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs(
|
|
// t,
|
|
// ctx,
|
|
// acEvts,
|
|
// uid,
|
|
// "",
|
|
// restoreCfg.Location)
|
|
|
|
// --- mail
|
|
cat = path.EmailCategory
|
|
userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs(
|
|
t,
|
|
ctx,
|
|
acMail,
|
|
uid,
|
|
"",
|
|
restoreCfg.Location,
|
|
api.MailInbox)
|
|
|
|
// then restore to the secondary user
|
|
|
|
uid = secondaryUserID
|
|
mb = evmock.NewBus()
|
|
secondCtr := count.New()
|
|
restoreCfg.ProtectedResource = uid
|
|
|
|
ro2, _ := prepNewTestRestoreOp(
|
|
t,
|
|
ctx,
|
|
bod.st,
|
|
bo.Results.BackupID,
|
|
mb,
|
|
secondCtr,
|
|
sel,
|
|
opts,
|
|
restoreCfg)
|
|
|
|
runAndCheckRestore(t, ctx, &ro2, mb, false)
|
|
|
|
var (
|
|
secondaryItemIDs = map[path.CategoryType]map[string]struct{}{}
|
|
secondaryCollisionKeys = map[path.CategoryType]map[string]string{}
|
|
)
|
|
|
|
// --- contacts
|
|
cat = path.ContactsCategory
|
|
secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs(
|
|
t,
|
|
ctx,
|
|
acCont,
|
|
uid,
|
|
"",
|
|
restoreCfg.Location)
|
|
|
|
// --- events
|
|
// cat = path.EventsCategory
|
|
// secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs(
|
|
// t,
|
|
// ctx,
|
|
// acEvts,
|
|
// uid,
|
|
// "",
|
|
// restoreCfg.Location)
|
|
|
|
// --- mail
|
|
cat = path.EmailCategory
|
|
secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs(
|
|
t,
|
|
ctx,
|
|
acMail,
|
|
uid,
|
|
"",
|
|
restoreCfg.Location,
|
|
api.MailInbox)
|
|
|
|
// compare restore results
|
|
for _, cat := range []path.CategoryType{path.ContactsCategory, path.EmailCategory, path.EventsCategory} {
|
|
assert.Equal(t, len(userItemIDs[cat]), len(secondaryItemIDs[cat]))
|
|
assert.ElementsMatch(t, maps.Keys(userCollisionKeys[cat]), maps.Keys(secondaryCollisionKeys[cat]))
|
|
}
|
|
}
|
|
|
|
func getCollKeysAndItemIDs(
|
|
t *testing.T,
|
|
ctx context.Context, //revive:disable-line:context-as-argument
|
|
cip clientItemPager,
|
|
userID, parentContainerID string,
|
|
containerNames ...string,
|
|
) (map[string]struct{}, map[string]string, string) {
|
|
var (
|
|
c graph.Container
|
|
err error
|
|
cID = parentContainerID
|
|
)
|
|
|
|
for _, cn := range containerNames {
|
|
c, err = cip.GetContainerByName(ctx, userID, cID, cn)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
cID = ptr.Val(c.GetId())
|
|
}
|
|
|
|
itemIDs, err := cip.GetItemIDsInContainer(ctx, userID, cID)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
collisionKeys, err := cip.GetItemsInContainerByCollisionKey(ctx, userID, cID)
|
|
require.NoError(t, err, clues.ToCore(err))
|
|
|
|
return itemIDs, collisionKeys, cID
|
|
}
|