fix overlapping bases after fallback union (#3118)
In a case where the current manifests and the fallback contain both distinct and overlapping categories, the end result contains multiple bases for the same category. Verifydistinctresons didn't catch this because it includes the resource owner. --- #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🐛 Bugfix #### Issue(s) * #2825 #### Test Plan - [x] ⚡ Unit test
This commit is contained in:
parent
d40830e01e
commit
482b690e06
@ -69,6 +69,7 @@ func (gc *GraphConnector) ProduceBackupCollections(
|
|||||||
colls, excludes, err := exchange.DataCollections(
|
colls, excludes, err := exchange.DataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
sels,
|
sels,
|
||||||
|
sels,
|
||||||
metadata,
|
metadata,
|
||||||
gc.credentials,
|
gc.credentials,
|
||||||
gc.UpdateStatus,
|
gc.UpdateStatus,
|
||||||
@ -95,7 +96,9 @@ func (gc *GraphConnector) ProduceBackupCollections(
|
|||||||
case selectors.ServiceOneDrive:
|
case selectors.ServiceOneDrive:
|
||||||
colls, excludes, err := onedrive.DataCollections(
|
colls, excludes, err := onedrive.DataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
sels, metadata,
|
sels,
|
||||||
|
sels,
|
||||||
|
metadata,
|
||||||
gc.credentials.AzureTenantID,
|
gc.credentials.AzureTenantID,
|
||||||
gc.itemClient,
|
gc.itemClient,
|
||||||
gc.Service,
|
gc.Service,
|
||||||
|
|||||||
@ -96,9 +96,12 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() {
|
|||||||
suite.Run(test.name, func() {
|
suite.Run(test.name, func() {
|
||||||
t := suite.T()
|
t := suite.T()
|
||||||
|
|
||||||
|
sel := test.getSelector(t)
|
||||||
|
|
||||||
collections, excludes, err := exchange.DataCollections(
|
collections, excludes, err := exchange.DataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
test.getSelector(t),
|
sel,
|
||||||
|
sel,
|
||||||
nil,
|
nil,
|
||||||
connector.credentials,
|
connector.credentials,
|
||||||
connector.UpdateStatus,
|
connector.UpdateStatus,
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common"
|
||||||
"github.com/alcionai/corso/src/internal/connector/exchange/api"
|
"github.com/alcionai/corso/src/internal/connector/exchange/api"
|
||||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||||
"github.com/alcionai/corso/src/internal/connector/support"
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
@ -162,6 +163,7 @@ func parseMetadataCollections(
|
|||||||
// Add iota to this call -> mail, contacts, calendar, etc.
|
// Add iota to this call -> mail, contacts, calendar, etc.
|
||||||
func DataCollections(
|
func DataCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
user common.IDNamer,
|
||||||
selector selectors.Selector,
|
selector selectors.Selector,
|
||||||
metadata []data.RestoreCollection,
|
metadata []data.RestoreCollection,
|
||||||
acct account.M365Config,
|
acct account.M365Config,
|
||||||
@ -175,7 +177,6 @@ func DataCollections(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
user = selector.DiscreteOwner
|
|
||||||
collections = []data.BackupCollection{}
|
collections = []data.BackupCollection{}
|
||||||
el = errs.Local()
|
el = errs.Local()
|
||||||
categories = map[path.CategoryType]struct{}{}
|
categories = map[path.CategoryType]struct{}{}
|
||||||
@ -214,7 +215,7 @@ func DataCollections(
|
|||||||
baseCols, err := graph.BaseCollections(
|
baseCols, err := graph.BaseCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct.AzureTenantID,
|
acct.AzureTenantID,
|
||||||
user,
|
user.ID(),
|
||||||
path.ExchangeService,
|
path.ExchangeService,
|
||||||
categories,
|
categories,
|
||||||
su,
|
su,
|
||||||
@ -248,7 +249,7 @@ func getterByType(ac api.Client, category path.CategoryType) (addedAndRemovedIte
|
|||||||
func createCollections(
|
func createCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
creds account.M365Config,
|
creds account.M365Config,
|
||||||
user string,
|
user common.IDNamer,
|
||||||
scope selectors.ExchangeScope,
|
scope selectors.ExchangeScope,
|
||||||
dps DeltaPaths,
|
dps DeltaPaths,
|
||||||
ctrlOpts control.Options,
|
ctrlOpts control.Options,
|
||||||
|
|||||||
@ -239,6 +239,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() {
|
|||||||
userID = tester.M365UserID(suite.T())
|
userID = tester.M365UserID(suite.T())
|
||||||
users = []string{userID}
|
users = []string{userID}
|
||||||
acct, err = tester.NewM365Account(suite.T()).M365Config()
|
acct, err = tester.NewM365Account(suite.T()).M365Config()
|
||||||
|
ss = selectors.Selector{}.SetDiscreteOwnerIDName(userID, userID)
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(suite.T(), err, clues.ToCore(err))
|
require.NoError(suite.T(), err, clues.ToCore(err))
|
||||||
@ -267,7 +268,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() {
|
|||||||
collections, err := createCollections(
|
collections, err := createCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
userID,
|
ss,
|
||||||
test.scope,
|
test.scope,
|
||||||
DeltaPaths{},
|
DeltaPaths{},
|
||||||
control.Options{},
|
control.Options{},
|
||||||
@ -299,6 +300,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
|
|||||||
userID = tester.M365UserID(suite.T())
|
userID = tester.M365UserID(suite.T())
|
||||||
users = []string{userID}
|
users = []string{userID}
|
||||||
acct, err = tester.NewM365Account(suite.T()).M365Config()
|
acct, err = tester.NewM365Account(suite.T()).M365Config()
|
||||||
|
ss = selectors.Selector{}.SetDiscreteOwnerIDName(userID, userID)
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(suite.T(), err, clues.ToCore(err))
|
require.NoError(suite.T(), err, clues.ToCore(err))
|
||||||
@ -337,7 +339,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
|
|||||||
collections, err := createCollections(
|
collections, err := createCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
userID,
|
ss,
|
||||||
test.scope,
|
test.scope,
|
||||||
DeltaPaths{},
|
DeltaPaths{},
|
||||||
control.Options{},
|
control.Options{},
|
||||||
@ -368,7 +370,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
|
|||||||
collections, err = createCollections(
|
collections, err = createCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
userID,
|
ss,
|
||||||
test.scope,
|
test.scope,
|
||||||
dps,
|
dps,
|
||||||
control.Options{},
|
control.Options{},
|
||||||
@ -403,6 +405,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
|
|||||||
t = suite.T()
|
t = suite.T()
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
users = []string{suite.user}
|
users = []string{suite.user}
|
||||||
|
ss = selectors.Selector{}.SetDiscreteOwnerIDName(suite.user, suite.user)
|
||||||
)
|
)
|
||||||
|
|
||||||
acct, err := tester.NewM365Account(t).M365Config()
|
acct, err := tester.NewM365Account(t).M365Config()
|
||||||
@ -414,7 +417,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
|
|||||||
collections, err := createCollections(
|
collections, err := createCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
suite.user,
|
ss,
|
||||||
sel.Scopes()[0],
|
sel.Scopes()[0],
|
||||||
DeltaPaths{},
|
DeltaPaths{},
|
||||||
control.Options{},
|
control.Options{},
|
||||||
@ -464,6 +467,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
|
|||||||
require.NoError(suite.T(), err, clues.ToCore(err))
|
require.NoError(suite.T(), err, clues.ToCore(err))
|
||||||
|
|
||||||
users := []string{suite.user}
|
users := []string{suite.user}
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName(suite.user, suite.user)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@ -487,7 +491,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
|
|||||||
edcs, err := createCollections(
|
edcs, err := createCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
suite.user,
|
ss,
|
||||||
test.scope,
|
test.scope,
|
||||||
DeltaPaths{},
|
DeltaPaths{},
|
||||||
control.Options{},
|
control.Options{},
|
||||||
@ -552,6 +556,8 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
|
|||||||
bdayID string
|
bdayID string
|
||||||
)
|
)
|
||||||
|
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName(suite.user, suite.user)
|
||||||
|
|
||||||
fn := func(gcf graph.CacheFolder) error {
|
fn := func(gcf graph.CacheFolder) error {
|
||||||
if ptr.Val(gcf.GetDisplayName()) == DefaultCalendar {
|
if ptr.Val(gcf.GetDisplayName()) == DefaultCalendar {
|
||||||
calID = ptr.Val(gcf.GetId())
|
calID = ptr.Val(gcf.GetId())
|
||||||
@ -599,7 +605,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
|
|||||||
collections, err := createCollections(
|
collections, err := createCollections(
|
||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
suite.user,
|
ss,
|
||||||
test.scope,
|
test.scope,
|
||||||
DeltaPaths{},
|
DeltaPaths{},
|
||||||
control.Options{},
|
control.Options{},
|
||||||
|
|||||||
@ -52,7 +52,7 @@ func PopulateExchangeContainerResolver(
|
|||||||
case path.EmailCategory:
|
case path.EmailCategory:
|
||||||
acm := ac.Mail()
|
acm := ac.Mail()
|
||||||
res = &mailFolderCache{
|
res = &mailFolderCache{
|
||||||
userID: qp.ResourceOwner,
|
userID: qp.ResourceOwner.ID(),
|
||||||
getter: acm,
|
getter: acm,
|
||||||
enumer: acm,
|
enumer: acm,
|
||||||
}
|
}
|
||||||
@ -61,7 +61,7 @@ func PopulateExchangeContainerResolver(
|
|||||||
case path.ContactsCategory:
|
case path.ContactsCategory:
|
||||||
acc := ac.Contacts()
|
acc := ac.Contacts()
|
||||||
res = &contactFolderCache{
|
res = &contactFolderCache{
|
||||||
userID: qp.ResourceOwner,
|
userID: qp.ResourceOwner.ID(),
|
||||||
getter: acc,
|
getter: acc,
|
||||||
enumer: acc,
|
enumer: acc,
|
||||||
}
|
}
|
||||||
@ -70,7 +70,7 @@ func PopulateExchangeContainerResolver(
|
|||||||
case path.EventsCategory:
|
case path.EventsCategory:
|
||||||
ecc := ac.Events()
|
ecc := ac.Events()
|
||||||
res = &eventCalendarCache{
|
res = &eventCalendarCache{
|
||||||
userID: qp.ResourceOwner,
|
userID: qp.ResourceOwner.ID(),
|
||||||
getter: ecc,
|
getter: ecc,
|
||||||
enumer: ecc,
|
enumer: ecc,
|
||||||
}
|
}
|
||||||
@ -113,7 +113,7 @@ func includeContainer(
|
|||||||
|
|
||||||
dirPath, err := pb.ToDataLayerExchangePathForCategory(
|
dirPath, err := pb.ToDataLayerExchangePathForCategory(
|
||||||
qp.Credentials.AzureTenantID,
|
qp.Credentials.AzureTenantID,
|
||||||
qp.ResourceOwner,
|
qp.ResourceOwner.ID(),
|
||||||
category,
|
category,
|
||||||
false)
|
false)
|
||||||
// Containers without a path (e.g. Root mail folder) always err here.
|
// Containers without a path (e.g. Root mail folder) always err here.
|
||||||
@ -126,7 +126,7 @@ func includeContainer(
|
|||||||
if loc != nil {
|
if loc != nil {
|
||||||
locPath, err = loc.ToDataLayerExchangePathForCategory(
|
locPath, err = loc.ToDataLayerExchangePathForCategory(
|
||||||
qp.Credentials.AzureTenantID,
|
qp.Credentials.AzureTenantID,
|
||||||
qp.ResourceOwner,
|
qp.ResourceOwner.ID(),
|
||||||
category,
|
category,
|
||||||
false)
|
false)
|
||||||
// Containers without a path (e.g. Root mail folder) always err here.
|
// Containers without a path (e.g. Root mail folder) always err here.
|
||||||
|
|||||||
@ -108,7 +108,7 @@ func filterContainersAndFillCollections(
|
|||||||
|
|
||||||
ictx = clues.Add(ictx, "previous_path", prevPath)
|
ictx = clues.Add(ictx, "previous_path", prevPath)
|
||||||
|
|
||||||
added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ictx, qp.ResourceOwner, cID, prevDelta)
|
added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ictx, qp.ResourceOwner.ID(), cID, prevDelta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !graph.IsErrDeletedInFlight(err) {
|
if !graph.IsErrDeletedInFlight(err) {
|
||||||
el.AddRecoverable(clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
|
el.AddRecoverable(clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
|
||||||
@ -130,7 +130,7 @@ func filterContainersAndFillCollections(
|
|||||||
}
|
}
|
||||||
|
|
||||||
edc := NewCollection(
|
edc := NewCollection(
|
||||||
qp.ResourceOwner,
|
qp.ResourceOwner.ID(),
|
||||||
currPath,
|
currPath,
|
||||||
prevPath,
|
prevPath,
|
||||||
locPath,
|
locPath,
|
||||||
@ -189,7 +189,7 @@ func filterContainersAndFillCollections(
|
|||||||
}
|
}
|
||||||
|
|
||||||
edc := NewCollection(
|
edc := NewCollection(
|
||||||
qp.ResourceOwner,
|
qp.ResourceOwner.ID(),
|
||||||
nil, // marks the collection as deleted
|
nil, // marks the collection as deleted
|
||||||
prevPath,
|
prevPath,
|
||||||
nil, // tombstones don't need a location
|
nil, // tombstones don't need a location
|
||||||
@ -208,7 +208,7 @@ func filterContainersAndFillCollections(
|
|||||||
|
|
||||||
col, err := graph.MakeMetadataCollection(
|
col, err := graph.MakeMetadataCollection(
|
||||||
qp.Credentials.AzureTenantID,
|
qp.Credentials.AzureTenantID,
|
||||||
qp.ResourceOwner,
|
qp.ResourceOwner.ID(),
|
||||||
path.ExchangeService,
|
path.ExchangeService,
|
||||||
qp.Category,
|
qp.Category,
|
||||||
[]graph.MetadataCollectionEntry{
|
[]graph.MetadataCollectionEntry{
|
||||||
|
|||||||
@ -116,11 +116,12 @@ func (suite *ServiceIteratorsSuite) SetupSuite() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
|
func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName("user_id", "user_id")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
userID = "user_id"
|
|
||||||
qp = graph.QueryParams{
|
qp = graph.QueryParams{
|
||||||
Category: path.EmailCategory, // doesn't matter which one we use.
|
Category: path.EmailCategory, // doesn't matter which one we use.
|
||||||
ResourceOwner: userID,
|
ResourceOwner: ss,
|
||||||
Credentials: suite.creds,
|
Credentials: suite.creds,
|
||||||
}
|
}
|
||||||
statusUpdater = func(*support.ConnectorOperationStatus) {}
|
statusUpdater = func(*support.ConnectorOperationStatus) {}
|
||||||
@ -435,11 +436,12 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_repea
|
|||||||
ctx, flush := tester.NewContext()
|
ctx, flush := tester.NewContext()
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName("user_id", "user_id")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
userID = "user_id"
|
|
||||||
qp = graph.QueryParams{
|
qp = graph.QueryParams{
|
||||||
Category: path.EmailCategory, // doesn't matter which one we use.
|
Category: path.EmailCategory, // doesn't matter which one we use.
|
||||||
ResourceOwner: userID,
|
ResourceOwner: ss,
|
||||||
Credentials: suite.creds,
|
Credentials: suite.creds,
|
||||||
}
|
}
|
||||||
statusUpdater = func(*support.ConnectorOperationStatus) {}
|
statusUpdater = func(*support.ConnectorOperationStatus) {}
|
||||||
@ -454,6 +456,9 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_repea
|
|||||||
resolver = newMockResolver(container1)
|
resolver = newMockResolver(container1)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
require.Equal(t, "user_id", qp.ResourceOwner.ID(), qp.ResourceOwner)
|
||||||
|
require.Equal(t, "user_id", qp.ResourceOwner.Name(), qp.ResourceOwner)
|
||||||
|
|
||||||
collections := map[string]data.BackupCollection{}
|
collections := map[string]data.BackupCollection{}
|
||||||
|
|
||||||
err := filterContainersAndFillCollections(
|
err := filterContainersAndFillCollections(
|
||||||
@ -514,13 +519,15 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_repea
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_incrementals() {
|
func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_incrementals() {
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName("user_id", "user_id")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
userID = "user_id"
|
userID = "user_id"
|
||||||
tenantID = suite.creds.AzureTenantID
|
tenantID = suite.creds.AzureTenantID
|
||||||
cat = path.EmailCategory // doesn't matter which one we use,
|
cat = path.EmailCategory // doesn't matter which one we use,
|
||||||
qp = graph.QueryParams{
|
qp = graph.QueryParams{
|
||||||
Category: cat,
|
Category: cat,
|
||||||
ResourceOwner: userID,
|
ResourceOwner: ss,
|
||||||
Credentials: suite.creds,
|
Credentials: suite.creds,
|
||||||
}
|
}
|
||||||
statusUpdater = func(*support.ConnectorOperationStatus) {}
|
statusUpdater = func(*support.ConnectorOperationStatus) {}
|
||||||
|
|||||||
@ -78,8 +78,7 @@ func MakeMetadataCollection(
|
|||||||
resourceOwner,
|
resourceOwner,
|
||||||
service,
|
service,
|
||||||
cat,
|
cat,
|
||||||
false,
|
false)
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "making metadata path")
|
return nil, clues.Wrap(err, "making metadata path")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -12,6 +12,7 @@ import (
|
|||||||
msgraphsdkgo "github.com/microsoftgraph/msgraph-sdk-go"
|
msgraphsdkgo "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
|
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
@ -38,7 +39,7 @@ func AllMetadataFileNames() []string {
|
|||||||
|
|
||||||
type QueryParams struct {
|
type QueryParams struct {
|
||||||
Category path.CategoryType
|
Category path.CategoryType
|
||||||
ResourceOwner string
|
ResourceOwner common.IDNamer
|
||||||
Credentials account.M365Config
|
Credentials account.M365Config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import (
|
|||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
"golang.org/x/exp/maps"
|
"golang.org/x/exp/maps"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common"
|
||||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||||
"github.com/alcionai/corso/src/internal/connector/support"
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
"github.com/alcionai/corso/src/internal/data"
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
@ -34,6 +35,7 @@ func (fm odFolderMatcher) Matches(dir string) bool {
|
|||||||
func DataCollections(
|
func DataCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
selector selectors.Selector,
|
selector selectors.Selector,
|
||||||
|
user common.IDNamer,
|
||||||
metadata []data.RestoreCollection,
|
metadata []data.RestoreCollection,
|
||||||
tenant string,
|
tenant string,
|
||||||
itemClient *http.Client,
|
itemClient *http.Client,
|
||||||
@ -49,7 +51,6 @@ func DataCollections(
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
el = errs.Local()
|
el = errs.Local()
|
||||||
user = selector.DiscreteOwner
|
|
||||||
categories = map[path.CategoryType]struct{}{}
|
categories = map[path.CategoryType]struct{}{}
|
||||||
collections = []data.BackupCollection{}
|
collections = []data.BackupCollection{}
|
||||||
allExcludes = map[string]map[string]struct{}{}
|
allExcludes = map[string]map[string]struct{}{}
|
||||||
@ -66,7 +67,7 @@ func DataCollections(
|
|||||||
nc := NewCollections(
|
nc := NewCollections(
|
||||||
itemClient,
|
itemClient,
|
||||||
tenant,
|
tenant,
|
||||||
user,
|
user.ID(),
|
||||||
OneDriveSource,
|
OneDriveSource,
|
||||||
odFolderMatcher{scope},
|
odFolderMatcher{scope},
|
||||||
service,
|
service,
|
||||||
@ -95,7 +96,7 @@ func DataCollections(
|
|||||||
baseCols, err := graph.BaseCollections(
|
baseCols, err := graph.BaseCollections(
|
||||||
ctx,
|
ctx,
|
||||||
tenant,
|
tenant,
|
||||||
user,
|
user.ID(),
|
||||||
path.OneDriveService,
|
path.OneDriveService,
|
||||||
categories,
|
categories,
|
||||||
su,
|
su,
|
||||||
|
|||||||
@ -844,9 +844,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
|
|||||||
|
|
||||||
// verify test data was populated, and track it for comparisons
|
// verify test data was populated, and track it for comparisons
|
||||||
for category, gen := range dataset {
|
for category, gen := range dataset {
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName(suite.user, suite.user)
|
||||||
|
|
||||||
qp := graph.QueryParams{
|
qp := graph.QueryParams{
|
||||||
Category: category,
|
Category: category,
|
||||||
ResourceOwner: suite.user,
|
ResourceOwner: ss,
|
||||||
Credentials: m365,
|
Credentials: m365,
|
||||||
}
|
}
|
||||||
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true))
|
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true))
|
||||||
@ -958,9 +960,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
|
|||||||
version.Backup,
|
version.Backup,
|
||||||
gen.dbf)
|
gen.dbf)
|
||||||
|
|
||||||
|
ss := selectors.Selector{}.SetDiscreteOwnerIDName(suite.user, suite.user)
|
||||||
|
|
||||||
qp := graph.QueryParams{
|
qp := graph.QueryParams{
|
||||||
Category: category,
|
Category: category,
|
||||||
ResourceOwner: suite.user,
|
ResourceOwner: ss,
|
||||||
Credentials: m365,
|
Credentials: m365,
|
||||||
}
|
}
|
||||||
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true))
|
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true))
|
||||||
|
|||||||
@ -93,20 +93,6 @@ func produceManifestsAndMetadata(
|
|||||||
return ms, nil, false, nil
|
return ms, nil, false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// We only need to check that we have 1:1 reason:base if we're doing an
|
|
||||||
// incremental with associated metadata. This ensures that we're only sourcing
|
|
||||||
// data from a single Point-In-Time (base) for each incremental backup.
|
|
||||||
//
|
|
||||||
// TODO(ashmrtn): This may need updating if we start sourcing item backup
|
|
||||||
// details from previous snapshots when using kopia-assisted incrementals.
|
|
||||||
if err := verifyDistinctBases(ctx, ms); err != nil {
|
|
||||||
logger.Ctx(ctx).With("error", err).Infow(
|
|
||||||
"unioned snapshot collision, falling back to full backup",
|
|
||||||
clues.In(ctx).Slice()...)
|
|
||||||
|
|
||||||
return ms, nil, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, man := range ms {
|
for _, man := range ms {
|
||||||
if len(man.IncompleteReason) > 0 {
|
if len(man.IncompleteReason) > 0 {
|
||||||
continue
|
continue
|
||||||
@ -234,6 +220,8 @@ func unionManifests(
|
|||||||
|
|
||||||
// backfill from the fallback where necessary
|
// backfill from the fallback where necessary
|
||||||
for _, m := range fallback {
|
for _, m := range fallback {
|
||||||
|
useReasons := []kopia.Reason{}
|
||||||
|
|
||||||
for _, r := range m.Reasons {
|
for _, r := range m.Reasons {
|
||||||
k := r.Service.String() + r.Category.String()
|
k := r.Service.String() + r.Category.String()
|
||||||
t := tups[k]
|
t := tups[k]
|
||||||
@ -245,6 +233,8 @@ func unionManifests(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
useReasons = append(useReasons, r)
|
||||||
|
|
||||||
if len(m.IncompleteReason) > 0 && t.incomplete == nil {
|
if len(m.IncompleteReason) > 0 && t.incomplete == nil {
|
||||||
t.incomplete = m
|
t.incomplete = m
|
||||||
} else if len(m.IncompleteReason) == 0 {
|
} else if len(m.IncompleteReason) == 0 {
|
||||||
@ -253,6 +243,10 @@ func unionManifests(
|
|||||||
|
|
||||||
tups[k] = t
|
tups[k] = t
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(m.IncompleteReason) == 0 && len(useReasons) > 0 {
|
||||||
|
m.Reasons = useReasons
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// collect the results into a single slice of manifests
|
// collect the results into a single slice of manifests
|
||||||
|
|||||||
@ -757,18 +757,20 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
|
|
||||||
table := []struct {
|
table := []struct {
|
||||||
name string
|
name string
|
||||||
main []testInput
|
man []testInput
|
||||||
fallback []testInput
|
fallback []testInput
|
||||||
reasons []kopia.Reason
|
reasons []kopia.Reason
|
||||||
fallbackReasons []kopia.Reason
|
fallbackReasons []kopia.Reason
|
||||||
categories []path.CategoryType
|
manCategories []path.CategoryType
|
||||||
|
fbCategories []path.CategoryType
|
||||||
assertErr assert.ErrorAssertionFunc
|
assertErr assert.ErrorAssertionFunc
|
||||||
expectManIDs []string
|
expectManIDs []string
|
||||||
expectNilMans bool
|
expectNilMans bool
|
||||||
|
expectReasons map[string][]path.CategoryType
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "only mans, no fallbacks",
|
name: "only mans, no fallbacks",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manComplete,
|
id: manComplete,
|
||||||
},
|
},
|
||||||
@ -777,8 +779,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
incomplete: true,
|
incomplete: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{manComplete, manIncomplete},
|
expectManIDs: []string{manComplete, manIncomplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.EmailCategory},
|
||||||
|
manIncomplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "no mans, only fallbacks",
|
name: "no mans, only fallbacks",
|
||||||
@ -791,12 +798,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
incomplete: true,
|
incomplete: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{fbComplete, fbIncomplete},
|
expectManIDs: []string{fbComplete, fbIncomplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
fbComplete: {path.EmailCategory},
|
||||||
|
fbIncomplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "complete mans and fallbacks",
|
name: "complete mans and fallbacks",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manComplete,
|
id: manComplete,
|
||||||
},
|
},
|
||||||
@ -806,12 +818,16 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
id: fbComplete,
|
id: fbComplete,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{manComplete},
|
expectManIDs: []string{manComplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "incomplete mans and fallbacks",
|
name: "incomplete mans and fallbacks",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manIncomplete,
|
id: manIncomplete,
|
||||||
incomplete: true,
|
incomplete: true,
|
||||||
@ -823,12 +839,16 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
incomplete: true,
|
incomplete: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{manIncomplete},
|
expectManIDs: []string{manIncomplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manIncomplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "complete and incomplete mans and fallbacks",
|
name: "complete and incomplete mans and fallbacks",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manComplete,
|
id: manComplete,
|
||||||
},
|
},
|
||||||
@ -846,12 +866,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
incomplete: true,
|
incomplete: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{manComplete, manIncomplete},
|
expectManIDs: []string{manComplete, manIncomplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.EmailCategory},
|
||||||
|
manIncomplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "incomplete mans, complete fallbacks",
|
name: "incomplete mans, complete fallbacks",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manIncomplete,
|
id: manIncomplete,
|
||||||
incomplete: true,
|
incomplete: true,
|
||||||
@ -862,12 +887,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
id: fbComplete,
|
id: fbComplete,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{fbComplete, manIncomplete},
|
expectManIDs: []string{fbComplete, manIncomplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
fbComplete: {path.EmailCategory},
|
||||||
|
manIncomplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "complete mans, incomplete fallbacks",
|
name: "complete mans, incomplete fallbacks",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manComplete,
|
id: manComplete,
|
||||||
},
|
},
|
||||||
@ -878,12 +908,16 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
incomplete: true,
|
incomplete: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory},
|
manCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
expectManIDs: []string{manComplete},
|
expectManIDs: []string{manComplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "complete mans, complete fallbacks, multiple reasons",
|
name: "complete mans, complete fallbacks, multiple reasons",
|
||||||
main: []testInput{
|
man: []testInput{
|
||||||
{
|
{
|
||||||
id: manComplete,
|
id: manComplete,
|
||||||
},
|
},
|
||||||
@ -893,8 +927,52 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
id: fbComplete,
|
id: fbComplete,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
categories: []path.CategoryType{path.EmailCategory, path.ContactsCategory},
|
manCategories: []path.CategoryType{path.EmailCategory, path.ContactsCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory, path.ContactsCategory},
|
||||||
expectManIDs: []string{manComplete},
|
expectManIDs: []string{manComplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.EmailCategory, path.ContactsCategory},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "complete mans, complete fallbacks, distinct reasons",
|
||||||
|
man: []testInput{
|
||||||
|
{
|
||||||
|
id: manComplete,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fallback: []testInput{
|
||||||
|
{
|
||||||
|
id: fbComplete,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
manCategories: []path.CategoryType{path.ContactsCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory},
|
||||||
|
expectManIDs: []string{manComplete, fbComplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.ContactsCategory},
|
||||||
|
fbComplete: {path.EmailCategory},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fb has superset of mans reasons",
|
||||||
|
man: []testInput{
|
||||||
|
{
|
||||||
|
id: manComplete,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fallback: []testInput{
|
||||||
|
{
|
||||||
|
id: fbComplete,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
manCategories: []path.CategoryType{path.ContactsCategory},
|
||||||
|
fbCategories: []path.CategoryType{path.EmailCategory, path.ContactsCategory, path.EventsCategory},
|
||||||
|
expectManIDs: []string{manComplete, fbComplete},
|
||||||
|
expectReasons: map[string][]path.CategoryType{
|
||||||
|
manComplete: {path.ContactsCategory},
|
||||||
|
fbComplete: {path.EmailCategory, path.EventsCategory},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, test := range table {
|
for _, test := range table {
|
||||||
@ -907,7 +985,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
mainReasons := []kopia.Reason{}
|
mainReasons := []kopia.Reason{}
|
||||||
fbReasons := []kopia.Reason{}
|
fbReasons := []kopia.Reason{}
|
||||||
|
|
||||||
for _, cat := range test.categories {
|
for _, cat := range test.manCategories {
|
||||||
mainReasons = append(
|
mainReasons = append(
|
||||||
mainReasons,
|
mainReasons,
|
||||||
kopia.Reason{
|
kopia.Reason{
|
||||||
@ -915,7 +993,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
Service: path.ExchangeService,
|
Service: path.ExchangeService,
|
||||||
Category: cat,
|
Category: cat,
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, cat := range test.fbCategories {
|
||||||
fbReasons = append(
|
fbReasons = append(
|
||||||
fbReasons,
|
fbReasons,
|
||||||
kopia.Reason{
|
kopia.Reason{
|
||||||
@ -927,7 +1007,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
|
|
||||||
mans := []*kopia.ManifestEntry{}
|
mans := []*kopia.ManifestEntry{}
|
||||||
|
|
||||||
for _, m := range test.main {
|
for _, m := range test.man {
|
||||||
incomplete := ""
|
incomplete := ""
|
||||||
if m.incomplete {
|
if m.incomplete {
|
||||||
incomplete = "ir"
|
incomplete = "ir"
|
||||||
@ -959,8 +1039,18 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
|
|||||||
assert.False(t, b, "no-metadata is forced for this test")
|
assert.False(t, b, "no-metadata is forced for this test")
|
||||||
|
|
||||||
manIDs := []string{}
|
manIDs := []string{}
|
||||||
|
|
||||||
for _, m := range mans {
|
for _, m := range mans {
|
||||||
manIDs = append(manIDs, string(m.ID))
|
manIDs = append(manIDs, string(m.ID))
|
||||||
|
|
||||||
|
reasons := test.expectReasons[string(m.ID)]
|
||||||
|
|
||||||
|
mrs := []path.CategoryType{}
|
||||||
|
for _, r := range m.Reasons {
|
||||||
|
mrs = append(mrs, r.Category)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, reasons, mrs)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.ElementsMatch(t, test.expectManIDs, manIDs)
|
assert.ElementsMatch(t, test.expectManIDs, manIDs)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user