add operations test for adv rest (#3783)

ads operations level tests for advanced restore
configuration on all three services.
Code is largely boilerplate between each service, but with just enough quirks that full consolidation would require excess jumping through hoops.

---

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🤖 Supportability/Tests

#### Issue(s)

* #3562

#### Test Plan

- [x] 💚 E2E
This commit is contained in:
Keepers 2023-07-16 21:56:08 -06:00 committed by GitHub
parent 09e5e9464a
commit 5a78f478a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 1414 additions and 57 deletions

View File

@ -239,7 +239,7 @@ func (w *conn) wrap() error {
defer w.mu.Unlock() defer w.mu.Unlock()
if w.refCount == 0 { if w.refCount == 0 {
return clues.New("conn already closed") return clues.New("conn not established or already closed")
} }
w.refCount++ w.refCount++

View File

@ -506,11 +506,11 @@ func (ms *ModelStore) DeleteWithModelStoreID(ctx context.Context, id manifest.ID
} }
opts := repo.WriteSessionOptions{Purpose: "ModelStoreDelete"} opts := repo.WriteSessionOptions{Purpose: "ModelStoreDelete"}
ctr := func(innerCtx context.Context, w repo.RepositoryWriter) error { cb := func(innerCtx context.Context, w repo.RepositoryWriter) error {
return w.DeleteManifest(innerCtx, id) return w.DeleteManifest(innerCtx, id)
} }
if err := repo.WriteSession(ctx, ms.c, opts, ctr); err != nil { if err := repo.WriteSession(ctx, ms.c, opts, cb); err != nil {
return clues.Wrap(err, "deleting model").WithClues(ctx) return clues.Wrap(err, "deleting model").WithClues(ctx)
} }

View File

@ -154,6 +154,12 @@ func restoreContact(
info := api.ContactInfo(item) info := api.ContactInfo(item)
info.Size = int64(len(body)) info.Size = int64(len(body))
if shouldDeleteOriginal {
ctr.Inc(count.CollisionReplace)
} else {
ctr.Inc(count.NewItemCreated)
}
return info, nil return info, nil
} }

View File

@ -90,6 +90,12 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
collisionKey := api.ContactCollisionKey(stub) collisionKey := api.ContactCollisionKey(stub)
type counts struct {
skip int64
replace int64
new int64
}
table := []struct { table := []struct {
name string name string
apiMock *contactRestoreMock apiMock *contactRestoreMock
@ -97,6 +103,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
onCollision control.CollisionPolicy onCollision control.CollisionPolicy
expectErr func(*testing.T, error) expectErr func(*testing.T, error)
expectMock func(*testing.T, *contactRestoreMock) expectMock func(*testing.T, *contactRestoreMock)
expectCounts counts
}{ }{
{ {
name: "no collision: skip", name: "no collision: skip",
@ -110,6 +117,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision: copy", name: "no collision: copy",
@ -123,6 +131,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision: replace", name: "no collision: replace",
@ -136,6 +145,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision: skip", name: "collision: skip",
@ -149,6 +159,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{1, 0, 0},
}, },
{ {
name: "collision: copy", name: "collision: copy",
@ -162,6 +173,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision: replace", name: "collision: replace",
@ -175,6 +187,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.True(t, m.calledDelete, "old item deleted") assert.True(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
{ {
name: "collision: replace - err already deleted", name: "collision: replace - err already deleted",
@ -188,6 +201,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.True(t, m.calledDelete, "old item deleted") assert.True(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
} }
for _, test := range table { for _, test := range table {
@ -197,6 +211,8 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
ctr := count.New()
_, err := restoreContact( _, err := restoreContact(
ctx, ctx,
test.apiMock, test.apiMock,
@ -206,10 +222,13 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() {
test.collisionMap, test.collisionMap,
test.onCollision, test.onCollision,
fault.New(true), fault.New(true),
count.New()) ctr)
test.expectErr(t, err) test.expectErr(t, err)
test.expectMock(t, test.apiMock) test.expectMock(t, test.apiMock)
assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips")
assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces")
assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items")
}) })
} }
} }

View File

@ -196,6 +196,12 @@ func restoreEvent(
info := api.EventInfo(event) info := api.EventInfo(event)
info.Size = int64(len(body)) info.Size = int64(len(body))
if shouldDeleteOriginal {
ctr.Inc(count.CollisionReplace)
} else {
ctr.Inc(count.NewItemCreated)
}
return info, nil return info, nil
} }

View File

@ -138,6 +138,12 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
collisionKey := api.EventCollisionKey(stub) collisionKey := api.EventCollisionKey(stub)
type counts struct {
skip int64
replace int64
new int64
}
table := []struct { table := []struct {
name string name string
apiMock *eventRestoreMock apiMock *eventRestoreMock
@ -145,6 +151,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
onCollision control.CollisionPolicy onCollision control.CollisionPolicy
expectErr func(*testing.T, error) expectErr func(*testing.T, error)
expectMock func(*testing.T, *eventRestoreMock) expectMock func(*testing.T, *eventRestoreMock)
expectCounts counts
}{ }{
{ {
name: "no collision: skip", name: "no collision: skip",
@ -158,6 +165,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision: copy", name: "no collision: copy",
@ -171,6 +179,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision: replace", name: "no collision: replace",
@ -184,6 +193,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision: skip", name: "collision: skip",
@ -197,6 +207,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{1, 0, 0},
}, },
{ {
name: "collision: copy", name: "collision: copy",
@ -210,6 +221,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision: replace", name: "collision: replace",
@ -223,6 +235,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.True(t, m.calledDelete, "old item deleted") assert.True(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
{ {
name: "collision: replace - err already deleted", name: "collision: replace - err already deleted",
@ -236,6 +249,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.True(t, m.calledDelete, "old item deleted") assert.True(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
} }
for _, test := range table { for _, test := range table {
@ -245,6 +259,8 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
ctr := count.New()
_, err := restoreEvent( _, err := restoreEvent(
ctx, ctx,
test.apiMock, test.apiMock,
@ -254,10 +270,13 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() {
test.collisionMap, test.collisionMap,
test.onCollision, test.onCollision,
fault.New(true), fault.New(true),
count.New()) ctr)
test.expectErr(t, err) test.expectErr(t, err)
test.expectMock(t, test.apiMock) test.expectMock(t, test.apiMock)
assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips")
assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces")
assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items")
}) })
} }
} }

View File

@ -174,6 +174,12 @@ func restoreMail(
size = int64(len(bc)) size = int64(len(bc))
} }
if shouldDeleteOriginal {
ctr.Inc(count.CollisionReplace)
} else {
ctr.Inc(count.NewItemCreated)
}
return api.MailInfo(msg, size), nil return api.MailInfo(msg, size), nil
} }

View File

@ -107,6 +107,12 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
collisionKey := api.MailCollisionKey(stub) collisionKey := api.MailCollisionKey(stub)
type counts struct {
skip int64
replace int64
new int64
}
table := []struct { table := []struct {
name string name string
apiMock *mailRestoreMock apiMock *mailRestoreMock
@ -114,6 +120,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
onCollision control.CollisionPolicy onCollision control.CollisionPolicy
expectErr func(*testing.T, error) expectErr func(*testing.T, error)
expectMock func(*testing.T, *mailRestoreMock) expectMock func(*testing.T, *mailRestoreMock)
expectCounts counts
}{ }{
{ {
name: "no collision: skip", name: "no collision: skip",
@ -127,6 +134,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision: copy", name: "no collision: copy",
@ -140,6 +148,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision: replace", name: "no collision: replace",
@ -153,6 +162,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision: skip", name: "collision: skip",
@ -166,6 +176,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{1, 0, 0},
}, },
{ {
name: "collision: copy", name: "collision: copy",
@ -179,6 +190,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.False(t, m.calledDelete, "old item deleted") assert.False(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision: replace", name: "collision: replace",
@ -192,6 +204,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.True(t, m.calledDelete, "old item deleted") assert.True(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
{ {
name: "collision: replace - err already deleted", name: "collision: replace - err already deleted",
@ -205,6 +218,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledPost, "new item posted")
assert.True(t, m.calledDelete, "old item deleted") assert.True(t, m.calledDelete, "old item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
} }
for _, test := range table { for _, test := range table {
@ -214,6 +228,8 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
ctr := count.New()
_, err := restoreMail( _, err := restoreMail(
ctx, ctx,
test.apiMock, test.apiMock,
@ -223,10 +239,13 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() {
test.collisionMap, test.collisionMap,
test.onCollision, test.onCollision,
fault.New(true), fault.New(true),
count.New()) ctr)
test.expectErr(t, err) test.expectErr(t, err)
test.expectMock(t, test.apiMock) test.expectMock(t, test.apiMock)
assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips")
assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces")
assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items")
}) })
} }
} }

View File

@ -108,7 +108,7 @@ func ConsumeRestoreCollections(
restoreCfg.OnCollision, restoreCfg.OnCollision,
deets, deets,
errs, errs,
ctr.Local()) ctr)
metrics = support.CombineMetrics(metrics, temp) metrics = support.CombineMetrics(metrics, temp)

View File

@ -237,7 +237,7 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() {
newMWReturns(test.status, nil, test.providedErr)) newMWReturns(test.status, nil, test.providedErr))
mw.repeatReturn0 = true mw.repeatReturn0 = true
adpt, err := mockAdapter(suite.creds, mw, 15*time.Second) adpt, err := mockAdapter(suite.creds, mw, 25*time.Second)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// url doesn't fit the builder, but that shouldn't matter // url doesn't fit the builder, but that shouldn't matter

View File

@ -117,7 +117,7 @@ type GetItemsByCollisionKeyser interface {
GetItemsInContainerByCollisionKey( GetItemsInContainerByCollisionKey(
ctx context.Context, ctx context.Context,
driveID, containerID string, driveID, containerID string,
) (map[string]api.DriveCollisionItem, error) ) (map[string]api.DriveItemIDType, error)
} }
type NewItemContentUploader interface { type NewItemContentUploader interface {

View File

@ -164,7 +164,7 @@ func (h itemRestoreHandler) DeleteItemPermission(
func (h itemRestoreHandler) GetItemsInContainerByCollisionKey( func (h itemRestoreHandler) GetItemsInContainerByCollisionKey(
ctx context.Context, ctx context.Context,
driveID, containerID string, driveID, containerID string,
) (map[string]api.DriveCollisionItem, error) { ) (map[string]api.DriveItemIDType, error) {
m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -239,7 +239,7 @@ func (m GetsItemPermission) GetItemPermission(
type RestoreHandler struct { type RestoreHandler struct {
ItemInfo details.ItemInfo ItemInfo details.ItemInfo
CollisionKeyMap map[string]api.DriveCollisionItem CollisionKeyMap map[string]api.DriveItemIDType
CalledDeleteItem bool CalledDeleteItem bool
CalledDeleteItemOn string CalledDeleteItemOn string
@ -264,7 +264,7 @@ func (h *RestoreHandler) AugmentItemInfo(
func (h *RestoreHandler) GetItemsInContainerByCollisionKey( func (h *RestoreHandler) GetItemsInContainerByCollisionKey(
context.Context, context.Context,
string, string, string, string,
) (map[string]api.DriveCollisionItem, error) { ) (map[string]api.DriveItemIDType, error) {
return h.CollisionKeyMap, nil return h.CollisionKeyMap, nil
} }

View File

@ -38,7 +38,7 @@ const (
) )
type restoreCaches struct { type restoreCaches struct {
collisionKeyToItemID map[string]api.DriveCollisionItem collisionKeyToItemID map[string]api.DriveItemIDType
DriveIDToRootFolderID map[string]string DriveIDToRootFolderID map[string]string
Folders *folderCache Folders *folderCache
OldLinkShareIDToNewID map[string]string OldLinkShareIDToNewID map[string]string
@ -50,7 +50,7 @@ type restoreCaches struct {
func NewRestoreCaches() *restoreCaches { func NewRestoreCaches() *restoreCaches {
return &restoreCaches{ return &restoreCaches{
collisionKeyToItemID: map[string]api.DriveCollisionItem{}, collisionKeyToItemID: map[string]api.DriveItemIDType{},
DriveIDToRootFolderID: map[string]string{}, DriveIDToRootFolderID: map[string]string{},
Folders: NewFolderCache(), Folders: NewFolderCache(),
OldLinkShareIDToNewID: map[string]string{}, OldLinkShareIDToNewID: map[string]string{},
@ -478,7 +478,7 @@ func restoreV0File(
fibn data.FetchItemByNamer, fibn data.FetchItemByNamer,
restoreFolderID string, restoreFolderID string,
copyBuffer []byte, copyBuffer []byte,
collisionKeyToItemID map[string]api.DriveCollisionItem, collisionKeyToItemID map[string]api.DriveItemIDType,
itemData data.Stream, itemData data.Stream,
ctr *count.Bus, ctr *count.Bus,
) (details.ItemInfo, error) { ) (details.ItemInfo, error) {
@ -808,7 +808,7 @@ func restoreFile(
name string, name string,
itemData data.Stream, itemData data.Stream,
driveID, parentFolderID string, driveID, parentFolderID string,
collisionKeyToItemID map[string]api.DriveCollisionItem, collisionKeyToItemID map[string]api.DriveItemIDType,
copyBuffer []byte, copyBuffer []byte,
ctr *count.Bus, ctr *count.Bus,
) (string, details.ItemInfo, error) { ) (string, details.ItemInfo, error) {
@ -826,7 +826,7 @@ func restoreFile(
var ( var (
item = newItem(name, false) item = newItem(name, false)
collisionKey = api.DriveItemCollisionKey(item) collisionKey = api.DriveItemCollisionKey(item)
collision api.DriveCollisionItem collision api.DriveItemIDType
shouldDeleteOriginal bool shouldDeleteOriginal bool
) )
@ -937,6 +937,12 @@ func restoreFile(
dii := ir.AugmentItemInfo(details.ItemInfo{}, newItem, written, nil) dii := ir.AugmentItemInfo(details.ItemInfo{}, newItem, written, nil)
if shouldDeleteOriginal {
ctr.Inc(count.CollisionReplace)
} else {
ctr.Inc(count.NewItemCreated)
}
return ptr.Val(newItem.GetId()), dii, nil return ptr.Val(newItem.GetId()), dii, nil
} }

View File

@ -329,47 +329,57 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() {
func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
const mndiID = "mndi-id" const mndiID = "mndi-id"
type counts struct {
skip int64
replace int64
new int64
}
table := []struct { table := []struct {
name string name string
collisionKeys map[string]api.DriveCollisionItem collisionKeys map[string]api.DriveItemIDType
onCollision control.CollisionPolicy onCollision control.CollisionPolicy
deleteErr error deleteErr error
expectSkipped assert.BoolAssertionFunc expectSkipped assert.BoolAssertionFunc
expectMock func(*testing.T, *mock.RestoreHandler) expectMock func(*testing.T, *mock.RestoreHandler)
expectCounts counts
}{ }{
{ {
name: "no collision, copy", name: "no collision, copy",
collisionKeys: map[string]api.DriveCollisionItem{}, collisionKeys: map[string]api.DriveItemIDType{},
onCollision: control.Copy, onCollision: control.Copy,
expectSkipped: assert.False, expectSkipped: assert.False,
expectMock: func(t *testing.T, rh *mock.RestoreHandler) { expectMock: func(t *testing.T, rh *mock.RestoreHandler) {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision, replace", name: "no collision, replace",
collisionKeys: map[string]api.DriveCollisionItem{}, collisionKeys: map[string]api.DriveItemIDType{},
onCollision: control.Replace, onCollision: control.Replace,
expectSkipped: assert.False, expectSkipped: assert.False,
expectMock: func(t *testing.T, rh *mock.RestoreHandler) { expectMock: func(t *testing.T, rh *mock.RestoreHandler) {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "no collision, skip", name: "no collision, skip",
collisionKeys: map[string]api.DriveCollisionItem{}, collisionKeys: map[string]api.DriveItemIDType{},
onCollision: control.Skip, onCollision: control.Skip,
expectSkipped: assert.False, expectSkipped: assert.False,
expectMock: func(t *testing.T, rh *mock.RestoreHandler) { expectMock: func(t *testing.T, rh *mock.RestoreHandler) {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision, copy", name: "collision, copy",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: {ItemID: mndiID}, mock.DriveItemFileName: {ItemID: mndiID},
}, },
onCollision: control.Copy, onCollision: control.Copy,
@ -378,10 +388,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "collision, replace", name: "collision, replace",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: {ItemID: mndiID}, mock.DriveItemFileName: {ItemID: mndiID},
}, },
onCollision: control.Replace, onCollision: control.Replace,
@ -391,10 +402,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.True(t, rh.CalledDeleteItem, "new item deleted") assert.True(t, rh.CalledDeleteItem, "new item deleted")
assert.Equal(t, mndiID, rh.CalledDeleteItemOn, "deleted the correct item") assert.Equal(t, mndiID, rh.CalledDeleteItemOn, "deleted the correct item")
}, },
expectCounts: counts{0, 1, 0},
}, },
{ {
name: "collision, replace - err already deleted", name: "collision, replace - err already deleted",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: {ItemID: "smarf"}, mock.DriveItemFileName: {ItemID: "smarf"},
}, },
onCollision: control.Replace, onCollision: control.Replace,
@ -404,10 +416,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.True(t, rh.CalledDeleteItem, "new item deleted") assert.True(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 1, 0},
}, },
{ {
name: "collision, skip", name: "collision, skip",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: {ItemID: mndiID}, mock.DriveItemFileName: {ItemID: mndiID},
}, },
onCollision: control.Skip, onCollision: control.Skip,
@ -416,10 +429,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.False(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{1, 0, 0},
}, },
{ {
name: "file-folder collision, copy", name: "file-folder collision, copy",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: { mock.DriveItemFileName: {
ItemID: mndiID, ItemID: mndiID,
IsFolder: true, IsFolder: true,
@ -431,10 +445,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "file-folder collision, replace", name: "file-folder collision, replace",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: { mock.DriveItemFileName: {
ItemID: mndiID, ItemID: mndiID,
IsFolder: true, IsFolder: true,
@ -446,10 +461,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{0, 0, 1},
}, },
{ {
name: "file-folder collision, skip", name: "file-folder collision, skip",
collisionKeys: map[string]api.DriveCollisionItem{ collisionKeys: map[string]api.DriveItemIDType{
mock.DriveItemFileName: { mock.DriveItemFileName: {
ItemID: mndiID, ItemID: mndiID,
IsFolder: true, IsFolder: true,
@ -461,6 +477,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
assert.False(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledPostItem, "new item posted")
assert.False(t, rh.CalledDeleteItem, "new item deleted") assert.False(t, rh.CalledDeleteItem, "new item deleted")
}, },
expectCounts: counts{1, 0, 0},
}, },
} }
for _, test := range table { for _, test := range table {
@ -491,6 +508,8 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
dp, err := path.ToDrivePath(dpp) dp, err := path.ToDrivePath(dpp)
require.NoError(t, err) require.NoError(t, err)
ctr := count.New()
_, skip, err := restoreItem( _, skip, err := restoreItem(
ctx, ctx,
rh, rh,
@ -511,10 +530,14 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData),
}, },
nil, nil,
count.New()) ctr)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
test.expectSkipped(t, skip) test.expectSkipped(t, skip)
test.expectMock(t, rh) test.expectMock(t, rh)
assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips")
assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces")
assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items")
}) })
} }
} }

View File

@ -190,7 +190,7 @@ func (h libraryRestoreHandler) DeleteItemPermission(
func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey( func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey(
ctx context.Context, ctx context.Context,
driveID, containerID string, driveID, containerID string,
) (map[string]api.DriveCollisionItem, error) { ) (map[string]api.DriveItemIDType, error) {
m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -28,6 +28,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -79,7 +80,7 @@ func NewBackupOperation(
bus events.Eventer, bus events.Eventer,
) (BackupOperation, error) { ) (BackupOperation, error) {
op := BackupOperation{ op := BackupOperation{
operation: newOperation(opts, bus, kw, sw), operation: newOperation(opts, bus, count.New(), kw, sw),
ResourceOwner: owner, ResourceOwner: owner,
Selectors: selector, Selectors: selector,
Version: "v0", Version: "v0",

View File

@ -22,7 +22,7 @@ func getBackupAndDetailsFromID(
) (*backup.Backup, *details.Details, error) { ) (*backup.Backup, *details.Details, error) {
bup, err := ms.GetBackup(ctx, backupID) bup, err := ms.GetBackup(ctx, backupID)
if err != nil { if err != nil {
return nil, nil, clues.Wrap(err, "getting backup") return nil, nil, clues.Stack(err)
} }
deets, err := getDetailsFromBackup(ctx, bup, detailsStore, errs) deets, err := getDetailsFromBackup(ctx, bup, detailsStore, errs)

View File

@ -13,6 +13,7 @@ import (
"github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/stats"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
) )
// MaintenanceOperation wraps an operation with restore-specific props. // MaintenanceOperation wraps an operation with restore-specific props.
@ -36,7 +37,7 @@ func NewMaintenanceOperation(
bus events.Eventer, bus events.Eventer,
) (MaintenanceOperation, error) { ) (MaintenanceOperation, error) {
op := MaintenanceOperation{ op := MaintenanceOperation{
operation: newOperation(opts, bus, kw, nil), operation: newOperation(opts, bus, count.New(), kw, nil),
mOpts: mOpts, mOpts: mOpts,
} }

View File

@ -63,13 +63,14 @@ type operation struct {
func newOperation( func newOperation(
opts control.Options, opts control.Options,
bus events.Eventer, bus events.Eventer,
ctr *count.Bus,
kw *kopia.Wrapper, kw *kopia.Wrapper,
sw *store.Wrapper, sw *store.Wrapper,
) operation { ) operation {
return operation{ return operation{
CreatedAt: time.Now(), CreatedAt: time.Now(),
Errors: fault.New(opts.FailureHandling == control.FailFast), Errors: fault.New(opts.FailureHandling == control.FailFast),
Counter: count.New(), Counter: ctr,
Options: opts, Options: opts,
bus: bus, bus: bus,

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/store" "github.com/alcionai/corso/src/pkg/store"
) )
@ -25,7 +26,7 @@ func TestOperationSuite(t *testing.T) {
func (suite *OperationSuite) TestNewOperation() { func (suite *OperationSuite) TestNewOperation() {
t := suite.T() t := suite.T()
op := newOperation(control.Defaults(), events.Bus{}, nil, nil) op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil)
assert.Greater(t, op.CreatedAt, time.Time{}) assert.Greater(t, op.CreatedAt, time.Time{})
} }
@ -45,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() {
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
err := newOperation(control.Defaults(), events.Bus{}, test.kw, test.sw).validate() err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate()
test.errCheck(suite.T(), err, clues.ToCore(err)) test.errCheck(suite.T(), err, clues.ToCore(err))
}) })
} }

View File

@ -65,9 +65,10 @@ func NewRestoreOperation(
sel selectors.Selector, sel selectors.Selector,
restoreCfg control.RestoreConfig, restoreCfg control.RestoreConfig,
bus events.Eventer, bus events.Eventer,
ctr *count.Bus,
) (RestoreOperation, error) { ) (RestoreOperation, error) {
op := RestoreOperation{ op := RestoreOperation{
operation: newOperation(opts, bus, kw, sw), operation: newOperation(opts, bus, ctr, kw, sw),
acct: acct, acct: acct,
BackupID: backupID, BackupID: backupID,
RestoreCfg: control.EnsureRestoreConfigDefaults(ctx, restoreCfg), RestoreCfg: control.EnsureRestoreConfigDefaults(ctx, restoreCfg),

View File

@ -30,6 +30,7 @@ import (
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
@ -118,7 +119,8 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
"foo", "foo",
selectors.Selector{DiscreteOwner: "test"}, selectors.Selector{DiscreteOwner: "test"},
restoreCfg, restoreCfg,
evmock.NewBus()) evmock.NewBus(),
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
op.Errors.Fail(test.fail) op.Errors.Fail(test.fail)
@ -258,7 +260,8 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
"backup-id", "backup-id",
selectors.Selector{DiscreteOwner: "test"}, selectors.Selector{DiscreteOwner: "test"},
restoreCfg, restoreCfg,
evmock.NewBus()) evmock.NewBus(),
count.New())
test.errCheck(t, err, clues.ToCore(err)) test.errCheck(t, err, clues.ToCore(err))
}) })
} }
@ -427,7 +430,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
bup.backupID, bup.backupID,
test.getSelector(t, bup.selectorResourceOwners), test.getSelector(t, bup.selectorResourceOwners),
test.restoreCfg, test.restoreCfg,
mb) mb,
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ds, err := ro.Run(ctx) ds, err := ro.Run(ctx)
@ -481,7 +485,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() {
"backupID", "backupID",
rsel.Selector, rsel.Selector,
restoreCfg, restoreCfg,
mb) mb,
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ds, err := ro.Run(ctx) ds, err := ro.Run(ctx)

View File

@ -26,8 +26,11 @@ import (
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details"
deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
@ -868,3 +871,404 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
}) })
} }
} }
type ExchangeRestoreIntgSuite struct {
tester.Suite
its intgTesterSetup
}
func TestExchangeRestoreIntgSuite(t *testing.T) {
suite.Run(t, &ExchangeRestoreIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}),
})
}
func (suite *ExchangeRestoreIntgSuite) SetupSuite() {
suite.its = newIntegrationTesterSetup(suite.T())
}
func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
// a backup is required to run restores
baseSel := selectors.NewExchangeBackup([]string{suite.its.userID})
baseSel.Include(
// events cannot be run, for the same reason as incremental backups: the user needs
// to have their account recycled.
// base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()),
baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()),
baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
baseSel.DiscreteOwner = suite.its.userID
var (
mb = evmock.NewBus()
opts = control.Defaults()
)
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup)
defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false)
rsel, err := baseSel.ToExchangeRestore()
require.NoError(t, err, clues.ToCore(err))
var (
restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore")
sel = rsel.Selector
userID = sel.ID()
cIDs = map[path.CategoryType]string{
path.ContactsCategory: "",
path.EmailCategory: "",
path.EventsCategory: "",
}
collKeys = map[path.CategoryType]map[string]string{}
countContactsInRestore int
acCont = suite.its.ac.Contacts()
contactIDs map[string]struct{}
countEmailsInRestore int
acMail = suite.its.ac.Mail()
mailIDs map[string]struct{}
countItemsInRestore int
// countEventsInRestore int
// acEvts = suite.its.ac.Events()
// eventIDs = []string{}
)
// initial restore
suite.Run("baseline", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr1 := count.New()
restoreCfg.OnCollision = control.Copy
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr1,
sel,
opts,
restoreCfg)
runAndCheckRestore(t, ctx, &ro, mb, false)
// get all files in folder, use these as the base
// set of files to compare against.
// --- contacts
contGC, err := acCont.GetContainerByName(ctx, userID, "", restoreCfg.Location)
require.NoError(t, err, clues.ToCore(err))
cIDs[path.ContactsCategory] = ptr.Val(contGC.GetId())
collKeys[path.ContactsCategory], err = acCont.GetItemsInContainerByCollisionKey(
ctx,
userID,
cIDs[path.ContactsCategory])
require.NoError(t, err, clues.ToCore(err))
countContactsInRestore = len(collKeys[path.ContactsCategory])
t.Log(countContactsInRestore, "contacts restored")
contactIDs, err = acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory])
require.NoError(t, err, clues.ToCore(err))
// --- events
// gc, err = acEvts.GetContainerByName(ctx, userID, "", restoreCfg.Location)
// require.NoError(t, err, clues.ToCore(err))
// restoredContainerID[path.EventsCategory] = ptr.Val(gc.GetId())
// collKeys[path.EventsCategory], err = acEvts.GetItemsInContainerByCollisionKey(
// ctx,
// userID,
// cIDs[path.EventsCategory])
// require.NoError(t, err, clues.ToCore(err))
// countEventsInRestore = len(collKeys[path.EventsCategory])
// t.Log(countContactsInRestore, "events restored")
mailGC, err := acMail.GetContainerByName(ctx, userID, api.MsgFolderRoot, restoreCfg.Location)
require.NoError(t, err, clues.ToCore(err))
mailGC, err = acMail.GetContainerByName(ctx, userID, ptr.Val(mailGC.GetId()), api.MailInbox)
require.NoError(t, err, clues.ToCore(err))
cIDs[path.EmailCategory] = ptr.Val(mailGC.GetId())
// --- mail
collKeys[path.EmailCategory], err = acMail.GetItemsInContainerByCollisionKey(
ctx,
userID,
cIDs[path.EmailCategory])
require.NoError(t, err, clues.ToCore(err))
countEmailsInRestore = len(collKeys[path.EmailCategory])
t.Log(countContactsInRestore, "emails restored")
mailIDs, err = acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory])
require.NoError(t, err, clues.ToCore(err))
countItemsInRestore = countContactsInRestore + countEmailsInRestore // + countEventsInRestore
checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore)
})
// skip restore
suite.Run("skip collisions", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr2 := count.New()
restoreCfg.OnCollision = control.Skip
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr2,
sel,
opts,
restoreCfg)
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
assert.Zero(
t,
len(deets.Entries),
"no items should have been restored")
checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0)
// --- contacts
// get all files in folder, use these as the base
// set of files to compare against.
result := filterCollisionKeyResults(
t,
ctx,
userID,
cIDs[path.ContactsCategory],
GetItemsInContainerByCollisionKeyer[string](acCont),
collKeys[path.ContactsCategory])
currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory])
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, contactIDs, currentContactIDs, "ids are equal")
// --- events
// m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory])
// maps.Copy(result, m)
// --- mail
m := filterCollisionKeyResults(
t,
ctx,
userID,
cIDs[path.EmailCategory],
GetItemsInContainerByCollisionKeyer[string](acMail),
collKeys[path.EmailCategory])
maps.Copy(result, m)
currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory])
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, mailIDs, currentMailIDs, "ids are equal")
assert.Len(t, result, 0, "no new items should get added")
})
// replace restore
suite.Run("replace collisions", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr3 := count.New()
restoreCfg.OnCollision = control.Replace
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr3,
sel,
opts,
restoreCfg)
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
filtEnts := []details.Entry{}
for _, e := range deets.Entries {
if e.Folder == nil {
filtEnts = append(filtEnts, e)
}
}
assert.Len(
t,
filtEnts,
countItemsInRestore,
"every item should have been replaced")
// --- contacts
result := filterCollisionKeyResults(
t,
ctx,
userID,
cIDs[path.ContactsCategory],
GetItemsInContainerByCollisionKeyer[string](acCont),
collKeys[path.ContactsCategory])
currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory])
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, len(contactIDs), len(currentContactIDs), "count of ids are equal")
for orig := range contactIDs {
assert.NotContains(t, currentContactIDs, orig, "original item should not exist after replacement")
}
contactIDs = currentContactIDs
// --- events
// m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory])
// maps.Copy(result, m)
// --- mail
m := filterCollisionKeyResults(
t,
ctx,
userID,
cIDs[path.EmailCategory],
GetItemsInContainerByCollisionKeyer[string](acMail),
collKeys[path.EmailCategory])
maps.Copy(result, m)
checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0)
currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory])
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, len(mailIDs), len(currentMailIDs), "count of ids are equal")
for orig := range mailIDs {
assert.NotContains(t, currentMailIDs, orig, "original item should not exist after replacement")
}
mailIDs = currentMailIDs
assert.Len(t, result, 0, "all items should have been replaced")
})
// copy restore
suite.Run("copy collisions", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr4 := count.New()
restoreCfg.OnCollision = control.Copy
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr4,
sel,
opts,
restoreCfg)
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
filtEnts := []details.Entry{}
for _, e := range deets.Entries {
if e.Folder == nil {
filtEnts = append(filtEnts, e)
}
}
assert.Len(
t,
filtEnts,
countItemsInRestore,
"every item should have been copied")
checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore)
result := filterCollisionKeyResults(
t,
ctx,
userID,
cIDs[path.ContactsCategory],
GetItemsInContainerByCollisionKeyer[string](acCont),
collKeys[path.ContactsCategory])
currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory])
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, 2*len(contactIDs), len(currentContactIDs), "count of ids should be double from before")
assert.Subset(t, maps.Keys(currentContactIDs), maps.Keys(contactIDs), "original item should exist after copy")
// m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory])
// maps.Copy(result, m)
m := filterCollisionKeyResults(
t,
ctx,
userID,
cIDs[path.EmailCategory],
GetItemsInContainerByCollisionKeyer[string](acMail),
collKeys[path.EmailCategory])
maps.Copy(result, m)
currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory])
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, 2*len(mailIDs), len(currentMailIDs), "count of ids should be double from before")
assert.Subset(t, maps.Keys(currentMailIDs), maps.Keys(mailIDs), "original item should exist after copy")
// TODO: we have the option of modifying copy creations in exchange
// so that the results don't collide. But we haven't made that
// decision yet.
assert.Len(t, result, 0, "no items should have been added as copies")
})
}

View File

@ -215,10 +215,10 @@ func runAndCheckBackup(
expectStatus, expectStatus,
bo.Status) bo.Status)
require.Less(t, 0, bo.Results.ItemsWritten) require.NotZero(t, bo.Results.ItemsWritten)
assert.Less(t, 0, bo.Results.ItemsRead, "count of items read") assert.NotZero(t, bo.Results.ItemsRead, "count of items read")
assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") assert.NotZero(t, bo.Results.BytesRead, "bytes read")
assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") assert.NotZero(t, bo.Results.BytesUploaded, "bytes uploaded")
assert.Equal(t, 1, bo.Results.ResourceOwners, "count of resource owners") assert.Equal(t, 1, bo.Results.ResourceOwners, "count of resource owners")
assert.NoError(t, bo.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure())) assert.NoError(t, bo.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure()))
assert.Empty(t, bo.Errors.Recovered(), "incremental recoverable/iteration errors") assert.Empty(t, bo.Errors.Recovered(), "incremental recoverable/iteration errors")

View File

@ -32,6 +32,8 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
@ -961,3 +963,290 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() {
} }
} }
} }
type OneDriveRestoreIntgSuite struct {
tester.Suite
its intgTesterSetup
}
func TestOneDriveRestoreIntgSuite(t *testing.T) {
suite.Run(t, &OneDriveRestoreIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}),
})
}
func (suite *OneDriveRestoreIntgSuite) SetupSuite() {
suite.its = newIntegrationTesterSetup(suite.T())
}
func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() {
sel := selectors.NewOneDriveBackup([]string{suite.its.userID})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = suite.its.userID
runDriveRestoreWithAdvancedOptions(
suite.T(),
suite,
suite.its.ac,
sel.Selector,
suite.its.userDriveID,
suite.its.userDriveRootFolderID)
}
func runDriveRestoreWithAdvancedOptions(
t *testing.T,
suite tester.Suite,
ac api.Client,
sel selectors.Selector, // both Restore and Backup types work.
driveID, rootFolderID string,
) {
ctx, flush := tester.NewContext(t)
defer flush()
// a backup is required to run restores
var (
mb = evmock.NewBus()
opts = control.Defaults()
)
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup)
defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false)
var (
restoreCfg = ctrlTD.DefaultRestoreConfig("drive_adv_restore")
containerID string
countItemsInRestore int
collKeys = map[string]api.DriveItemIDType{}
fileIDs map[string]api.DriveItemIDType
acd = ac.Drives()
)
// initial restore
suite.Run("baseline", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr := count.New()
restoreCfg.OnCollision = control.Copy
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr,
sel,
opts,
restoreCfg)
runAndCheckRestore(t, ctx, &ro, mb, false)
// get all files in folder, use these as the base
// set of files to compare against.
contGC, err := acd.GetFolderByName(ctx, driveID, rootFolderID, restoreCfg.Location)
require.NoError(t, err, clues.ToCore(err))
// the folder containing the files is a child of the folder created by the restore.
contGC, err = acd.GetFolderByName(ctx, driveID, ptr.Val(contGC.GetId()), selTD.TestFolderName)
require.NoError(t, err, clues.ToCore(err))
containerID = ptr.Val(contGC.GetId())
collKeys, err = acd.GetItemsInContainerByCollisionKey(
ctx,
driveID,
containerID)
require.NoError(t, err, clues.ToCore(err))
countItemsInRestore = len(collKeys)
checkRestoreCounts(t, ctr, 0, 0, countItemsInRestore)
fileIDs, err = acd.GetItemIDsInContainer(ctx, driveID, containerID)
require.NoError(t, err, clues.ToCore(err))
})
// skip restore
suite.Run("skip collisions", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr := count.New()
restoreCfg.OnCollision = control.Skip
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr,
sel,
opts,
restoreCfg)
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
checkRestoreCounts(t, ctr, countItemsInRestore, 0, 0)
assert.Zero(
t,
len(deets.Entries),
"no items should have been restored")
// get all files in folder, use these as the base
// set of files to compare against.
result := filterCollisionKeyResults(
t,
ctx,
driveID,
containerID,
GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd),
collKeys)
assert.Len(t, result, 0, "no new items should get added")
currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, fileIDs, currentFileIDs, "ids are equal")
})
// replace restore
suite.Run("replace collisions", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr := count.New()
restoreCfg.OnCollision = control.Replace
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr,
sel,
opts,
restoreCfg)
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
filtEnts := []details.Entry{}
for _, e := range deets.Entries {
if e.Folder == nil {
filtEnts = append(filtEnts, e)
}
}
checkRestoreCounts(t, ctr, 0, countItemsInRestore, 0)
assert.Len(
t,
filtEnts,
countItemsInRestore,
"every item should have been replaced")
result := filterCollisionKeyResults(
t,
ctx,
driveID,
containerID,
GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd),
collKeys)
assert.Len(t, result, 0, "all items should have been replaced")
for k, v := range result {
assert.NotEqual(t, v, collKeys[k], "replaced items should have new IDs")
}
currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, len(fileIDs), len(currentFileIDs), "count of ids ids are equal")
for orig := range fileIDs {
assert.NotContains(t, currentFileIDs, orig, "original item should not exist after replacement")
}
fileIDs = currentFileIDs
})
// copy restore
suite.Run("copy collisions", func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
mb := evmock.NewBus()
ctr := count.New()
restoreCfg.OnCollision = control.Copy
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
ctr,
sel,
opts,
restoreCfg)
deets := runAndCheckRestore(t, ctx, &ro, mb, false)
filtEnts := []details.Entry{}
for _, e := range deets.Entries {
if e.Folder == nil {
filtEnts = append(filtEnts, e)
}
}
checkRestoreCounts(t, ctr, 0, 0, countItemsInRestore)
assert.Len(
t,
filtEnts,
countItemsInRestore,
"every item should have been copied")
result := filterCollisionKeyResults(
t,
ctx,
driveID,
containerID,
GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd),
collKeys)
assert.Len(t, result, len(collKeys), "all items should have been added as copies")
currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, 2*len(fileIDs), len(currentFileIDs), "count of ids should be double from before")
assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy")
})
}

View File

@ -0,0 +1,279 @@
package test_test
import (
"context"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/m365"
"github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/streamstore"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/storage"
"github.com/alcionai/corso/src/pkg/store"
)
type restoreOpDependencies struct {
acct account.Account
ctrl *m365.Controller
kms *kopia.ModelStore
kw *kopia.Wrapper
sel selectors.Selector
sss streamstore.Streamer
st storage.Storage
sw *store.Wrapper
closer func()
}
func (rod *restoreOpDependencies) close(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
) {
if rod.closer != nil {
rod.closer()
}
if rod.kw != nil {
err := rod.kw.Close(ctx)
assert.NoErrorf(t, err, "kw close: %+v", clues.ToCore(err))
}
if rod.kms != nil {
err := rod.kw.Close(ctx)
assert.NoErrorf(t, err, "kms close: %+v", clues.ToCore(err))
}
}
// prepNewTestRestoreOp generates all clients required to run a restore operation,
// returning both a restore operation created with those clients, as well as
// the clients themselves.
func prepNewTestRestoreOp(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
backupStore storage.Storage,
backupID model.StableID,
bus events.Eventer,
ctr *count.Bus,
sel selectors.Selector,
opts control.Options,
restoreCfg control.RestoreConfig,
) (
operations.RestoreOperation,
*restoreOpDependencies,
) {
var (
rod = &restoreOpDependencies{
acct: tconfig.NewM365Account(t),
st: backupStore,
}
k = kopia.NewConn(rod.st)
)
err := k.Connect(ctx, repository.Options{})
require.NoError(t, err, clues.ToCore(err))
// kopiaRef comes with a count of 1 and Wrapper bumps it again
// we're so safe to close here.
defer func() {
err := k.Close(ctx)
assert.NoErrorf(t, err, "k close: %+v", clues.ToCore(err))
}()
rod.kw, err = kopia.NewWrapper(k)
if !assert.NoError(t, err, clues.ToCore(err)) {
return operations.RestoreOperation{}, rod
}
rod.kms, err = kopia.NewModelStore(k)
if !assert.NoError(t, err, clues.ToCore(err)) {
return operations.RestoreOperation{}, rod
}
rod.sw = store.NewKopiaStore(rod.kms)
connectorResource := resource.Users
if sel.Service == selectors.ServiceSharePoint {
connectorResource = resource.Sites
}
rod.ctrl, rod.sel = ControllerWithSelector(
t,
ctx,
rod.acct,
connectorResource,
sel,
nil,
rod.close)
ro := newTestRestoreOp(
t,
ctx,
rod,
backupID,
bus,
ctr,
opts,
restoreCfg)
rod.sss = streamstore.NewStreamer(
rod.kw,
rod.acct.ID(),
rod.sel.PathService())
return ro, rod
}
// newTestRestoreOp accepts the clients required to compose a restore operation, plus
// any other metadata, and uses them to generate a new restore operation. This
// allows restore chains to utilize the same temp directory and configuration
// details.
func newTestRestoreOp(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
rod *restoreOpDependencies,
backupID model.StableID,
bus events.Eventer,
ctr *count.Bus,
opts control.Options,
restoreCfg control.RestoreConfig,
) operations.RestoreOperation {
rod.ctrl.IDNameLookup = idname.NewCache(map[string]string{rod.sel.ID(): rod.sel.Name()})
ro, err := operations.NewRestoreOperation(
ctx,
opts,
rod.kw,
rod.sw,
rod.ctrl,
rod.acct,
backupID,
rod.sel,
restoreCfg,
bus,
ctr)
if !assert.NoError(t, err, clues.ToCore(err)) {
rod.close(t, ctx)
t.FailNow()
}
return ro
}
func runAndCheckRestore(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
ro *operations.RestoreOperation,
mb *evmock.Bus,
acceptNoData bool,
) *details.Details {
deets, err := ro.Run(ctx)
require.NoError(t, err, clues.ToCore(err))
require.NotEmpty(t, ro.Results, "the restore had non-zero results")
require.NotNil(t, deets, "restore details")
expectStatus := []operations.OpStatus{operations.Completed}
if acceptNoData {
expectStatus = append(expectStatus, operations.NoData)
}
require.Contains(
t,
expectStatus,
ro.Status,
"restore doesn't match expectation, wanted any of %v, got %s",
expectStatus,
ro.Status)
assert.NoError(t, ro.Errors.Failure(), "non-recoverable error", clues.ToCore(ro.Errors.Failure()))
if assert.Empty(t, ro.Errors.Recovered(), "recoverable/iteration errors") {
allErrs := ro.Errors.Errors()
for i, err := range allErrs.Recovered {
t.Log("recovered from test err", i, err)
}
}
assert.NotZero(t, ro.Results.ItemsRead, "count of items read")
assert.NotZero(t, ro.Results.BytesRead, "bytes read")
assert.Equal(t, 1, ro.Results.ResourceOwners, "count of resource owners")
assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events")
assert.Equal(t, 1, mb.TimesCalled[events.RestoreEnd], "restore-end events")
return deets
}
type GetItemsInContainerByCollisionKeyer[T any] interface {
GetItemsInContainerByCollisionKey(
ctx context.Context,
userID, containerID string,
) (map[string]T, error)
}
func filterCollisionKeyResults[T any](
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
protectedResourceID, containerID string,
giicbck GetItemsInContainerByCollisionKeyer[T],
filterOut map[string]T,
) map[string]T {
m, err := giicbck.GetItemsInContainerByCollisionKey(
ctx,
protectedResourceID,
containerID)
require.NoError(t, err, clues.ToCore(err))
for k := range filterOut {
delete(m, k)
}
return m
}
func checkRestoreCounts(
t *testing.T,
ctr *count.Bus,
expectSkips, expectReplaces, expectNew int,
) {
t.Log("counted values", ctr.Values())
t.Log("counted totals", ctr.TotalValues())
if expectSkips >= 0 {
assert.Equal(
t,
int64(expectSkips),
ctr.Total(count.CollisionSkip),
"count of collisions resolved by skip")
}
if expectReplaces >= 0 {
assert.Equal(
t,
int64(expectReplaces),
ctr.Total(count.CollisionReplace),
"count of collisions resolved by replace")
}
if expectNew >= 0 {
assert.Equal(
t,
int64(expectNew),
ctr.Total(count.NewItemCreated),
"count of new items or collisions resolved by copy")
}
}

View File

@ -177,3 +177,35 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() {
} }
} }
} }
type SharePointRestoreIntgSuite struct {
tester.Suite
its intgTesterSetup
}
func TestSharePointRestoreIntgSuite(t *testing.T) {
suite.Run(t, &SharePointRestoreIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}),
})
}
func (suite *SharePointRestoreIntgSuite) SetupSuite() {
suite.its = newIntegrationTesterSetup(suite.T())
}
func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() {
sel := selectors.NewSharePointBackup([]string{suite.its.userID})
sel.Include(selTD.SharePointBackupFolderScope(sel))
sel.Filter(sel.Library("documents"))
sel.DiscreteOwner = suite.its.siteID
runDriveRestoreWithAdvancedOptions(
suite.T(),
suite,
suite.its.ac,
sel.Selector,
suite.its.siteDriveID,
suite.its.siteDriveRootFolderID)
}

View File

@ -3,5 +3,10 @@ package count
type key string type key string
const ( const (
CollisionSkip key = "collision-skip" // NewItemCreated should be used for non-skip, non-replace,
// non-meta item creation counting. IE: use it specifically
// for counting new items (no collision) or copied items.
NewItemCreated key = "new-item-created"
CollisionReplace key = "collision-replace"
CollisionSkip key = "collision-skip"
) )

View File

@ -26,6 +26,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
rep "github.com/alcionai/corso/src/pkg/control/repository" rep "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -370,7 +371,8 @@ func (r repository) NewRestore(
model.StableID(backupID), model.StableID(backupID),
sel, sel,
restoreCfg, restoreCfg,
r.Bus) r.Bus,
count.New())
} }
func (r repository) NewMaintenance( func (r repository) NewMaintenance(

View File

@ -2,8 +2,10 @@ package testdata
import "github.com/alcionai/corso/src/pkg/selectors" import "github.com/alcionai/corso/src/pkg/selectors"
const TestFolderName = "test"
// OneDriveBackupFolderScope is the standard folder scope that should be used // OneDriveBackupFolderScope is the standard folder scope that should be used
// in integration backups with onedrive. // in integration backups with onedrive.
func OneDriveBackupFolderScope(sel *selectors.OneDriveBackup) []selectors.OneDriveScope { func OneDriveBackupFolderScope(sel *selectors.OneDriveBackup) []selectors.OneDriveScope {
return sel.Folders([]string{"test"}, selectors.PrefixMatch()) return sel.Folders([]string{TestFolderName}, selectors.PrefixMatch())
} }

View File

@ -7,5 +7,5 @@ import (
// SharePointBackupFolderScope is the standard folder scope that should be used // SharePointBackupFolderScope is the standard folder scope that should be used
// in integration backups with sharepoint. // in integration backups with sharepoint.
func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope { func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope {
return sel.LibraryFolders([]string{"test"}, selectors.PrefixMatch()) return sel.LibraryFolders([]string{TestFolderName}, selectors.PrefixMatch())
} }

View File

@ -166,6 +166,27 @@ func (c Contacts) GetItemsInContainerByCollisionKey(
return m, nil return m, nil
} }
func (c Contacts) GetItemIDsInContainer(
ctx context.Context,
userID, containerID string,
) (map[string]struct{}, error) {
ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewContactsPager(userID, containerID, "id")
items, err := enumerateItems(ctx, pager)
if err != nil {
return nil, graph.Wrap(ctx, err, "enumerating contacts")
}
m := map[string]struct{}{}
for _, item := range items {
m[ptr.Val(item.GetId())] = struct{}{}
}
return m, nil
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// item ID pager // item ID pager
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -83,3 +83,47 @@ func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsInContainerByCollision
assert.Truef(t, ok, "expected results to contain collision key: %s", e) assert.Truef(t, ok, "expected results to contain collision key: %s", e)
} }
} }
func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsIDsInContainer() {
t := suite.T()
ac := suite.its.ac.Contacts()
ctx, flush := tester.NewContext(t)
defer flush()
container, err := ac.GetContainerByID(ctx, suite.its.userID, api.DefaultContacts)
require.NoError(t, err, clues.ToCore(err))
msgs, err := ac.Stable.
Client().
Users().
ByUserId(suite.its.userID).
ContactFolders().
ByContactFolderId(ptr.Val(container.GetId())).
Contacts().
Get(ctx, nil)
require.NoError(t, err, clues.ToCore(err))
ms := msgs.GetValue()
expect := map[string]struct{}{}
for _, m := range ms {
expect[ptr.Val(m.GetId())] = struct{}{}
}
results, err := suite.its.ac.Contacts().
GetItemIDsInContainer(ctx, suite.its.userID, api.DefaultContacts)
require.NoError(t, err, clues.ToCore(err))
require.Less(t, 0, len(results), "requires at least one result")
require.Equal(t, len(expect), len(results), "must have same count of items")
for _, k := range expect {
t.Log("expects key", k)
}
for k := range results {
t.Log("results key", k)
}
assert.Equal(t, expect, results)
}

View File

@ -67,7 +67,7 @@ func (p *driveItemPageCtrl) setNext(nextLink string) {
p.builder = drives.NewItemItemsItemChildrenRequestBuilder(nextLink, p.gs.Adapter()) p.builder = drives.NewItemItemsItemChildrenRequestBuilder(nextLink, p.gs.Adapter())
} }
type DriveCollisionItem struct { type DriveItemIDType struct {
ItemID string ItemID string
IsFolder bool IsFolder bool
} }
@ -75,7 +75,7 @@ type DriveCollisionItem struct {
func (c Drives) GetItemsInContainerByCollisionKey( func (c Drives) GetItemsInContainerByCollisionKey(
ctx context.Context, ctx context.Context,
driveID, containerID string, driveID, containerID string,
) (map[string]DriveCollisionItem, error) { ) (map[string]DriveItemIDType, error) {
ctx = clues.Add(ctx, "container_id", containerID) ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewDriveItemPager(driveID, containerID, idAnd("name")...) pager := c.NewDriveItemPager(driveID, containerID, idAnd("name")...)
@ -84,10 +84,34 @@ func (c Drives) GetItemsInContainerByCollisionKey(
return nil, graph.Wrap(ctx, err, "enumerating drive items") return nil, graph.Wrap(ctx, err, "enumerating drive items")
} }
m := map[string]DriveCollisionItem{} m := map[string]DriveItemIDType{}
for _, item := range items { for _, item := range items {
m[DriveItemCollisionKey(item)] = DriveCollisionItem{ m[DriveItemCollisionKey(item)] = DriveItemIDType{
ItemID: ptr.Val(item.GetId()),
IsFolder: item.GetFolder() != nil,
}
}
return m, nil
}
func (c Drives) GetItemIDsInContainer(
ctx context.Context,
driveID, containerID string,
) (map[string]DriveItemIDType, error) {
ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewDriveItemPager(driveID, containerID, idAnd("file", "folder")...)
items, err := enumerateItems(ctx, pager)
if err != nil {
return nil, graph.Wrap(ctx, err, "enumerating contacts")
}
m := map[string]DriveItemIDType{}
for _, item := range items {
m[ptr.Val(item.GetId())] = DriveItemIDType{
ItemID: ptr.Val(item.GetId()), ItemID: ptr.Val(item.GetId()),
IsFolder: item.GetFolder() != nil, IsFolder: item.GetFolder() != nil,
} }

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
@ -68,7 +69,7 @@ func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ims := items.GetValue() ims := items.GetValue()
expect := make([]api.DriveCollisionItem, 0, len(ims)) expect := make([]api.DriveItemIDType, 0, len(ims))
assert.NotEmptyf( assert.NotEmptyf(
t, t,
@ -103,3 +104,77 @@ func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey()
}) })
} }
} }
func (suite *DrivePagerIntgSuite) TestDrives_GetItemIDsInContainer() {
table := []struct {
name string
driveID string
rootFolderID string
}{
{
name: "user drive",
driveID: suite.its.userDriveID,
rootFolderID: suite.its.userDriveRootFolderID,
},
{
name: "site drive",
driveID: suite.its.siteDriveID,
rootFolderID: suite.its.siteDriveRootFolderID,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
t.Log("drive", test.driveID)
t.Log("rootFolder", test.rootFolderID)
items, err := suite.its.ac.Stable.
Client().
Drives().
ByDriveId(test.driveID).
Items().
ByDriveItemId(test.rootFolderID).
Children().
Get(ctx, nil)
require.NoError(t, err, clues.ToCore(err))
igv := items.GetValue()
expect := map[string]api.DriveItemIDType{}
assert.NotEmptyf(
t,
igv,
"need at least one item to compare in user %s drive %s folder %s",
suite.its.userID, test.driveID, test.rootFolderID)
for _, itm := range igv {
expect[ptr.Val(itm.GetId())] = api.DriveItemIDType{
ItemID: ptr.Val(itm.GetId()),
IsFolder: itm.GetFolder() != nil,
}
}
results, err := suite.its.ac.
Drives().
GetItemIDsInContainer(ctx, test.driveID, test.rootFolderID)
require.NoError(t, err, clues.ToCore(err))
require.NotEmpty(t, results)
require.Equal(t, len(expect), len(results), "must have same count of items")
for k := range expect {
t.Log("expects key", k)
}
for k, v := range results {
t.Log("results key", k)
assert.NotEmpty(t, v, "all values should be populated")
}
assert.Equal(t, expect, results)
})
}
}

View File

@ -247,6 +247,27 @@ func (c Mail) GetItemsInContainerByCollisionKey(
return m, nil return m, nil
} }
func (c Mail) GetItemIDsInContainer(
ctx context.Context,
userID, containerID string,
) (map[string]struct{}, error) {
ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewMailPager(userID, containerID, "id")
items, err := enumerateItems(ctx, pager)
if err != nil {
return nil, graph.Wrap(ctx, err, "enumerating contacts")
}
m := map[string]struct{}{}
for _, item := range items {
m[ptr.Val(item.GetId())] = struct{}{}
}
return m, nil
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// delta item ID pager // delta item ID pager
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -4,6 +4,7 @@ import (
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -83,3 +84,50 @@ func (suite *MailPagerIntgSuite) TestMail_GetItemsInContainerByCollisionKey() {
assert.Truef(t, ok, "expected results to contain collision key: %s", e) assert.Truef(t, ok, "expected results to contain collision key: %s", e)
} }
} }
func (suite *MailPagerIntgSuite) TestMail_GetItemsIDsInContainer() {
t := suite.T()
ac := suite.its.ac.Mail()
ctx, flush := tester.NewContext(t)
defer flush()
config := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{
Top: ptr.To[int32](1000),
},
}
msgs, err := ac.Stable.
Client().
Users().
ByUserId(suite.its.userID).
MailFolders().
ByMailFolderId(api.MailInbox).
Messages().
Get(ctx, config)
require.NoError(t, err, clues.ToCore(err))
ms := msgs.GetValue()
expect := map[string]struct{}{}
for _, m := range ms {
expect[ptr.Val(m.GetId())] = struct{}{}
}
results, err := suite.its.ac.Mail().
GetItemIDsInContainer(ctx, suite.its.userID, api.MailInbox)
require.NoError(t, err, clues.ToCore(err))
require.Less(t, 0, len(results), "requires at least one result")
require.Equal(t, len(expect), len(results), "must have same count of items")
for k := range expect {
t.Log("expects key", k)
}
for k := range results {
t.Log("results key", k)
}
assert.Equal(t, expect, results)
}

View File

@ -2,7 +2,6 @@ package m365
import ( import (
"context" "context"
"fmt"
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -508,8 +507,6 @@ func (suite *DiscoveryIntgSuite) TestGetUserInfo() {
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
fmt.Printf("\n-----\n%+v\n-----\n", test.name)
t := suite.T() t := suite.T()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)