modifies list restore handler and handles lists advanced restore (#5024)

- includes GetList in list restore handler
- includes GetListsByCollisionKey in list restore handler
- updates mock list restore handler


#### Does this PR need a docs update or release note?
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

#### Issue(s)
#4754 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Hitesh Pattanayak 2024-01-23 16:18:47 +05:30 committed by GitHub
parent 90648ef564
commit dfb4a73f56
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 703 additions and 284 deletions

View File

@ -253,7 +253,7 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
cats: []string{"invalid category"},
expect: assert.Error,
},
// [TODO]: Uncomment when lists are enabled
// [TODO](hitesh): Uncomment when lists are enabled
// {
// name: "site with lists category",

View File

@ -61,6 +61,10 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
"--" + flags.FormatFN, flagsTD.FormatType,
@ -88,6 +92,10 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
assert.Equal(t, flagsTD.Archive, opts.ExportCfg.Archive)

View File

@ -12,18 +12,32 @@ const (
const (
LibraryFN = "library"
ListFN = "list"
ListModifiedAfterFN = "list-modified-after"
ListModifiedBeforeFN = "list-modified-before"
ListCreatedAfterFN = "list-created-after"
ListCreatedBeforeFN = "list-created-before"
PageFolderFN = "page-folder"
PageFN = "page"
SiteFN = "site" // site only accepts WebURL values
SiteIDFN = "site-id" // site-id accepts actual site ids
)
var (
LibraryFV string
ListFV []string
ListModifiedAfterFV string
ListModifiedBeforeFV string
ListCreatedAfterFV string
ListCreatedBeforeFV string
PageFolderFV []string
PageFV []string
SiteIDFV []string
WebURLFV []string
)
@ -68,8 +82,23 @@ func AddSharePointDetailsAndRestoreFlags(cmd *cobra.Command) {
fs.StringSliceVar(
&ListFV,
ListFN, nil,
"Select lists by name; accepts '"+Wildcard+"' to select all lists.")
cobra.CheckErr(fs.MarkHidden(ListFN))
"Select lists by name.")
fs.StringVar(
&ListModifiedAfterFV,
ListModifiedAfterFN, "",
"Select lists modified after this datetime.")
fs.StringVar(
&ListModifiedBeforeFV,
ListModifiedBeforeFN, "",
"Select lists modified before this datetime.")
fs.StringVar(
&ListCreatedAfterFV,
ListCreatedAfterFN, "",
"Select lists created after this datetime.")
fs.StringVar(
&ListCreatedBeforeFV,
ListCreatedBeforeFN, "",
"Select lists created before this datetime.")
// pages

View File

@ -60,6 +60,10 @@ var (
FileModifiedBeforeInput = "fileModifiedBefore"
ListsInput = []string{"listName1", "listName2"}
ListCreatedAfterInput = "listCreatedAfter"
ListCreatedBeforeInput = "listCreatedBefore"
ListModifiedAfterInput = "listModifiedAfter"
ListModifiedBeforeInput = "listModifiedBefore"
PageFolderInput = []string{"pageFolder1", "pageFolder2"}
PageInput = []string{"page1", "page2"}

View File

@ -60,6 +60,10 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
"--" + flags.CollisionsFN, flagsTD.Collisions,
@ -89,6 +93,10 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
assert.Equal(t, flagsTD.Collisions, opts.RestoreCfg.Collisions)

View File

@ -50,6 +50,10 @@ func validateCommonTimeFlags(opts any) error {
flags.FileCreatedBeforeFN,
flags.FileModifiedAfterFN,
flags.FileModifiedBeforeFN,
flags.ListCreatedAfterFN,
flags.ListCreatedBeforeFN,
flags.ListModifiedAfterFN,
flags.ListModifiedBeforeFN,
}
isFlagPopulated := func(opts any, flag string) bool {

View File

@ -26,6 +26,10 @@ type SharePointOpts struct {
FileModifiedBefore string
Lists []string
ListModifiedAfter string
ListModifiedBefore string
ListCreatedBefore string
ListCreatedAfter string
PageFolder []string
Page []string
@ -46,6 +50,14 @@ func (s SharePointOpts) GetFileTimeField(flag string) string {
return s.FileModifiedAfter
case flags.FileModifiedBeforeFN:
return s.FileModifiedBefore
case flags.ListModifiedAfterFN:
return s.ListModifiedAfter
case flags.ListModifiedBeforeFN:
return s.ListModifiedBefore
case flags.ListCreatedBeforeFN:
return s.ListCreatedBefore
case flags.ListCreatedAfterFN:
return s.ListCreatedAfter
default:
return ""
}
@ -65,6 +77,10 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
FileModifiedBefore: flags.FileModifiedBeforeFV,
Lists: flags.ListFV,
ListModifiedAfter: flags.ListModifiedAfterFV,
ListModifiedBefore: flags.ListModifiedBeforeFV,
ListCreatedAfter: flags.ListCreatedAfterFV,
ListCreatedBefore: flags.ListCreatedBeforeFV,
Page: flags.PageFV,
PageFolder: flags.PageFolderFV,
@ -238,4 +254,8 @@ func FilterSharePointRestoreInfoSelectors(
AddSharePointInfo(sel, opts.FileCreatedBefore, sel.CreatedBefore)
AddSharePointInfo(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
AddSharePointInfo(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
AddSharePointInfo(sel, opts.ListModifiedAfter, sel.ListModifiedAfter)
AddSharePointInfo(sel, opts.ListModifiedBefore, sel.ListModifiedBefore)
AddSharePointInfo(sel, opts.ListCreatedAfter, sel.ListCreatedAfter)
AddSharePointInfo(sel, opts.ListCreatedBefore, sel.ListCreatedBefore)
}

View File

@ -279,12 +279,20 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
FileCreatedBefore: dttm.Now(),
FileModifiedAfter: dttm.Now(),
FileModifiedBefore: dttm.Now(),
ListCreatedAfter: dttm.Now(),
ListCreatedBefore: dttm.Now(),
ListModifiedAfter: dttm.Now(),
ListModifiedBefore: dttm.Now(),
Populated: flags.PopulatedFlags{
flags.SiteFN: struct{}{},
flags.FileCreatedAfterFN: struct{}{},
flags.FileCreatedBeforeFN: struct{}{},
flags.FileModifiedAfterFN: struct{}{},
flags.FileModifiedBeforeFN: struct{}{},
flags.ListCreatedAfterFN: struct{}{},
flags.ListCreatedBeforeFN: struct{}{},
flags.ListModifiedAfterFN: struct{}{},
flags.ListModifiedBeforeFN: struct{}{},
},
},
expect: assert.NoError,
@ -350,6 +358,50 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
},
expect: assert.Error,
},
{
name: "invalid list created after",
backupID: "id",
opts: utils.SharePointOpts{
ListCreatedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.ListCreatedAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid list created before",
backupID: "id",
opts: utils.SharePointOpts{
ListCreatedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.ListCreatedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid list modified after",
backupID: "id",
opts: utils.SharePointOpts{
ListModifiedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.ListModifiedAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid list modified before",
backupID: "id",
opts: utils.SharePointOpts{
ListModifiedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.ListModifiedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -359,7 +411,7 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
}
}
// [TODO] uncomment the test cases once sharepoint list backup is enabled
// [TODO]hitesh uncomment the test cases once sharepoint list backup is enabled
func (suite *SharePointUtilsSuite) TestAddSharepointCategories() {
table := []struct {
name string

View File

@ -36,7 +36,10 @@ type getItemser interface {
type restoreHandler interface {
PostLister
PatchLister
DeleteLister
GetLister
GetListsByCollisionKeyser
}
type PostLister interface {
@ -48,9 +51,35 @@ type PostLister interface {
) (models.Listable, error)
}
type PatchLister interface {
PatchList(
ctx context.Context,
listID string,
list models.Listable,
) (models.Listable, error)
}
type DeleteLister interface {
DeleteList(
ctx context.Context,
listID string,
) error
}
type GetLister interface {
GetList(
ctx context.Context,
listID string,
) (models.Listable, *details.SharePointInfo, error)
}
type GetListsByCollisionKeyser interface {
// GetListsByCollisionKey looks up all lists currently in
// the site, and returns them in a map[collisionKey]listID.
// The collision key is displayName of the list
// which uniquely identifies the list.
// Collision key checks are used during restore to handle the on-
// collision restore configurations that cause the list restore to get
// skipped, replaced, or copied.
GetListsByCollisionKey(ctx context.Context) (map[string]string, error)
}

View File

@ -73,9 +73,28 @@ func (rh listsRestoreHandler) PostList(
return rh.ac.PostList(ctx, rh.protectedResource, listName, storedList, errs)
}
func (rh listsRestoreHandler) PatchList(
ctx context.Context,
listID string,
list models.Listable,
) (models.Listable, error) {
return rh.ac.PatchList(ctx, rh.protectedResource, listID, list)
}
func (rh listsRestoreHandler) DeleteList(
ctx context.Context,
listID string,
) error {
return rh.ac.DeleteList(ctx, rh.protectedResource, listID)
}
func (rh listsRestoreHandler) GetList(
ctx context.Context,
listID string,
) (models.Listable, *details.SharePointInfo, error) {
return rh.ac.GetListByID(ctx, rh.protectedResource, listID)
}
func (rh listsRestoreHandler) GetListsByCollisionKey(ctx context.Context) (map[string]string, error) {
return rh.ac.GetListsByCollisionKey(ctx, rh.protectedResource)
}

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
@ -94,21 +95,53 @@ func (lh *ListHandler) Check(t *testing.T, expected []string) {
}
type ListRestoreHandler struct {
List models.Listable
Err error
deleteListErr error
postListErr error
patchListErr error
}
func NewListRestoreHandler(deleteListErr error, postListErr, patchListErr error) *ListRestoreHandler {
return &ListRestoreHandler{
deleteListErr: deleteListErr,
postListErr: postListErr,
patchListErr: patchListErr,
}
}
func (lh *ListRestoreHandler) PostList(
ctx context.Context,
listName string,
storedListBytes []byte,
storedList models.Listable,
_ *fault.Bus,
) (models.Listable, error) {
ls, _ := api.ToListable(storedList, listName)
return ls, lh.postListErr
}
func (lh *ListRestoreHandler) PatchList(
ctx context.Context,
listID string,
list models.Listable,
) (models.Listable, error) {
return nil, lh.patchListErr
}
func (lh *ListRestoreHandler) GetList(
ctx context.Context,
listID string,
) (models.Listable, *details.SharePointInfo, error) {
ls := models.NewList()
ls.SetId(ptr.To(listID))
lh.List = ls
lh.List.SetDisplayName(ptr.To(listName))
return ls, api.ListToSPInfo(ls), nil
}
return lh.List, lh.Err
func (lh *ListRestoreHandler) DeleteList(_ context.Context, _ string) error {
return lh.deleteListErr
}
func (lh *ListRestoreHandler) GetListsByCollisionKey(ctx context.Context) (map[string]string, error) {
return map[string]string{}, nil
}
func StubLists(ids ...string) []models.Listable {

View File

@ -10,18 +10,15 @@ import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/m365/collection/drive"
betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/errs/core"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
@ -29,107 +26,6 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
)
// ConsumeRestoreCollections will restore the specified data collections into OneDrive
func ConsumeRestoreCollections(
ctx context.Context,
rcc inject.RestoreConsumerConfig,
ac api.Client,
backupDriveIDNames idname.Cacher,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Bus,
ctr *count.Bus,
) (*support.ControllerOperationStatus, error) {
var (
lrh = drive.NewSiteRestoreHandler(ac, rcc.Selector.PathService())
listsRh = NewListsRestoreHandler(rcc.ProtectedResource.ID(), ac.Lists())
restoreMetrics support.CollectionMetrics
caches = drive.NewRestoreCaches(backupDriveIDNames)
el = errs.Local()
)
err := caches.Populate(ctx, ac.Users(), ac.Groups(), lrh, rcc.ProtectedResource.ID(), errs)
if err != nil {
return nil, clues.Wrap(err, "initializing restore caches")
}
// Reorder collections so that the parents directories are created
// before the child directories; a requirement for permissions.
data.SortRestoreCollections(dcs)
// Iterate through the data collections and restore the contents of each
for _, dc := range dcs {
if el.Failure() != nil {
break
}
var (
err error
category = dc.FullPath().Category()
metrics support.CollectionMetrics
ictx = clues.Add(ctx,
"category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ProtectedResource()),
"full_path", dc.FullPath())
)
switch dc.FullPath().Category() {
case path.LibrariesCategory:
metrics, err = drive.RestoreCollection(
ictx,
lrh,
rcc,
dc,
caches,
deets,
control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem),
errs,
ctr)
case path.ListsCategory:
metrics, err = RestoreListCollection(
ictx,
listsRh,
dc,
rcc.RestoreConfig.Location,
deets,
errs)
case path.PagesCategory:
metrics, err = RestorePageCollection(
ictx,
ac.Stable,
dc,
rcc.RestoreConfig.Location,
deets,
errs)
default:
return nil, clues.Wrap(clues.New(category.String()), "category not supported").With("category", category)
}
restoreMetrics = support.CombineMetrics(restoreMetrics, metrics)
if err != nil {
el.AddRecoverable(ctx, err)
}
if errors.Is(err, context.Canceled) {
break
}
}
status := support.CreateStatus(
ctx,
support.Restore,
len(dcs),
restoreMetrics,
rcc.RestoreConfig.Location)
return status, el.Failure()
}
// restoreListItem utility function restores a List to the siteID.
// The name is changed to to {DestName}_{name}
// API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http
@ -138,12 +34,17 @@ func restoreListItem(
ctx context.Context,
rh restoreHandler,
itemData data.Item,
siteID, destName string,
siteID string,
restoreCfg control.RestoreConfig,
collisionKeyToItemID map[string]string,
ctr *count.Bus,
errs *fault.Bus,
) (details.ItemInfo, error) {
var (
dii = details.ItemInfo{}
itemID = itemData.ID()
destName = restoreCfg.Location
collisionPolicy = restoreCfg.OnCollision
)
ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreList", diagnostics.Label("item_uuid", itemData.ID()))
@ -161,25 +62,101 @@ func restoreListItem(
return dii, clues.WrapWC(ctx, err, "generating list from stored bytes")
}
newName := formatListsRestoreDestination(destName, itemID, storedList)
var (
collisionKey = api.ListCollisionKey(storedList)
collisionID string
restoredList models.Listable
newName = formatListsRestoreDestination(destName, itemID, storedList)
)
if id, ok := collisionKeyToItemID[collisionKey]; ok {
log := logger.Ctx(ctx).With("collision_key", clues.Hide(collisionKey))
log.Debug("item collision")
if collisionPolicy == control.Skip {
ctr.Inc(count.CollisionSkip)
log.Debug("skipping item with collision")
return dii, clues.Stack(core.ErrAlreadyExists)
}
collisionID = id
}
if collisionPolicy != control.Replace {
restoredList, err = rh.PostList(ctx, newName, storedList, errs)
if err != nil {
return dii, clues.Wrap(err, "restoring list")
}
} else {
restoredList, err = handleListReplace(
ctx,
collisionID,
storedList,
newName,
rh,
ctr,
errs)
if err != nil {
return dii, clues.Stack(err)
}
}
// Restore to List base to M365 back store
restoredList, err := rh.PostList(ctx, newName, storedList, errs)
if err != nil {
return dii, graph.Wrap(ctx, err, "restoring list")
}
dii.SharePoint = api.ListToSPInfo(restoredList)
return dii, nil
}
func handleListReplace(
ctx context.Context,
listID string,
listFromBackup models.Listable,
newName string,
rh restoreHandler,
ctr *count.Bus,
errs *fault.Bus,
) (models.Listable, error) {
restoredList, err := rh.PostList(
ctx,
newName,
listFromBackup,
errs)
if err != nil {
return nil, clues.WrapWC(ctx, err, "restoring list")
}
err = rh.DeleteList(ctx, listID)
if err != nil {
return nil, clues.WrapWC(ctx, err, "deleting collided list")
}
patchList := models.NewList()
patchList.SetDisplayName(listFromBackup.GetDisplayName())
_, err = rh.PatchList(
ctx,
ptr.Val(restoredList.GetId()),
patchList)
if err != nil {
return nil, clues.WrapWC(ctx, err, "patching list")
}
restoredList.SetDisplayName(listFromBackup.GetDisplayName())
ctr.Inc(count.CollisionReplace)
return restoredList, nil
}
func RestoreListCollection(
ctx context.Context,
rh restoreHandler,
dc data.RestoreCollection,
restoreContainerName string,
restoreCfg control.RestoreConfig,
deets *details.Builder,
collisionKeyToItemID map[string]string,
ctr *count.Bus,
errs *fault.Bus,
) (support.CollectionMetrics, error) {
ctx, end := diagnostics.Span(ctx, "m365:sharepoint:restoreListCollection", diagnostics.Label("path", dc.FullPath()))
@ -215,7 +192,9 @@ func RestoreListCollection(
rh,
itemData,
siteID,
restoreContainerName,
restoreCfg,
collisionKeyToItemID,
ctr,
errs)
if errors.Is(err, api.ErrSkippableListTemplate) {
// should never be encountered as lists with skippable template are not backed up

View File

@ -3,6 +3,7 @@ package site
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"testing"
@ -17,6 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/common/readers"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
siteMock "github.com/alcionai/corso/src/internal/m365/collection/site/mock"
spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
@ -26,6 +28,7 @@ import (
"github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/errs/core"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
@ -127,16 +130,30 @@ func (suite *SharePointRestoreSuite) TestListCollection_Restore() {
ctx, flush := tester.NewContext(t)
defer flush()
testName, lrh, destName, mockData := setupDependencies(
suite,
suite.ac,
suite.siteID,
suite.creds,
"genericList")
var (
listName = "MockListing"
listTemplate = "genericList"
restoreCfg = testdata.DefaultRestoreConfig("")
destName = restoreCfg.Location
lrh = NewListsRestoreHandler(suite.siteID, suite.ac.Lists())
service = createTestService(t, suite.creds)
list = stubList(listTemplate, listName)
mockData = generateListData(t, service, list)
)
deets, err := restoreListItem(ctx, lrh, mockData, suite.siteID, destName, fault.New(true))
restoreCfg.OnCollision = control.Copy
deets, err := restoreListItem(
ctx,
lrh,
mockData,
suite.siteID,
restoreCfg,
nil,
count.New(),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, fmt.Sprintf("%s_%s", destName, testName), deets.SharePoint.List.Name)
assert.Equal(t, fmt.Sprintf("%s_%s", destName, listName), deets.SharePoint.List.Name)
// Clean-Up
deleteList(ctx, t, suite.siteID, lrh, deets)
@ -148,37 +165,28 @@ func (suite *SharePointRestoreSuite) TestListCollection_Restore_invalidListTempl
ctx, flush := tester.NewContext(t)
defer flush()
var (
lrh = NewListsRestoreHandler(suite.siteID, suite.ac.Lists())
listName = "MockListing"
restoreCfg = testdata.DefaultRestoreConfig("")
service = createTestService(t, suite.creds)
)
restoreCfg.OnCollision = control.Copy
tests := []struct {
name string
getParams func() (listsRestoreHandler, string, *dataMock.Item)
list models.Listable
expect assert.ErrorAssertionFunc
}{
{
name: "list with template documentLibrary",
getParams: func() (listsRestoreHandler, string, *dataMock.Item) {
_, lrh, destName, mockData := setupDependencies(
suite,
suite.ac,
suite.siteID,
suite.creds,
api.DocumentLibraryListTemplate)
return lrh, destName, mockData
},
list: stubList(api.DocumentLibraryListTemplate, listName),
expect: assert.Error,
},
{
name: "list with template webTemplateExtensionsList",
getParams: func() (listsRestoreHandler, string, *dataMock.Item) {
_, lrh, destName, mockData := setupDependencies(
suite,
suite.ac,
suite.siteID,
suite.creds,
api.WebTemplateExtensionsListTemplate)
return lrh, destName, mockData
},
list: stubList(api.WebTemplateExtensionsListTemplate, listName),
expect: assert.Error,
},
}
@ -187,14 +195,16 @@ func (suite *SharePointRestoreSuite) TestListCollection_Restore_invalidListTempl
suite.Run(test.name, func() {
t := suite.T()
lrh, destName, mockData := test.getParams()
listData := generateListData(t, service, test.list)
_, err := restoreListItem(
ctx,
lrh,
mockData,
listData,
suite.siteID,
destName,
restoreCfg,
nil,
count.New(),
fault.New(false))
require.Error(t, err)
assert.Contains(t, err.Error(), api.ErrSkippableListTemplate.Error())
@ -202,6 +212,169 @@ func (suite *SharePointRestoreSuite) TestListCollection_Restore_invalidListTempl
}
}
func (suite *SharePointRestoreSuite) TestListCollection_RestoreInPlace_skip() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
var (
listName = "MockListing"
listTemplate = "genericList"
restoreCfg = testdata.DefaultRestoreConfig("")
lrh = NewListsRestoreHandler(suite.siteID, suite.ac.Lists())
service = createTestService(t, suite.creds)
list = stubList(listTemplate, listName)
newList = stubList(listTemplate, listName)
cl = count.New()
)
mockData := generateListData(t, service, list)
collisionKeyToItemID := map[string]string{
api.ListCollisionKey(newList): "some-list-id",
}
deets, err := restoreListItem(
ctx,
lrh,
mockData,
suite.siteID,
restoreCfg, // OnCollision is skip by default
collisionKeyToItemID,
cl,
fault.New(true))
require.Error(t, err, clues.ToCore(err))
assert.Equal(t, core.ErrAlreadyExists.Error(), err.Error())
assert.Empty(t, deets)
assert.Less(t, int64(0), cl.Get(count.CollisionSkip))
}
func (suite *SharePointRestoreSuite) TestListCollection_RestoreInPlace_copy() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
var (
listName = "MockListing"
listTemplate = "genericList"
listID = "some-list-id"
restoreCfg = testdata.DefaultRestoreConfig("")
service = createTestService(t, suite.creds)
policyToKey = map[control.CollisionPolicy]count.Key{
control.Replace: count.CollisionReplace,
control.Skip: count.CollisionSkip,
}
)
list := stubList(listTemplate, listName)
list.SetId(ptr.To(listID))
newList := stubList(listTemplate, listName)
newList.SetId(ptr.To(listID))
collisionKeyToItemID := map[string]string{
api.ListCollisionKey(newList): listID,
}
tests := []struct {
name string
lrh *siteMock.ListRestoreHandler
expectErr assert.ErrorAssertionFunc
collisionPolicy control.CollisionPolicy
expectCollisionCount int64
}{
{
name: "PostList fails for stored list",
lrh: siteMock.NewListRestoreHandler(
nil,
errors.New("failed to create list"),
nil),
collisionPolicy: control.Replace,
expectErr: assert.Error,
},
{
name: "DeleteList fails",
lrh: siteMock.NewListRestoreHandler(
errors.New("failed to delete list"),
nil,
nil),
collisionPolicy: control.Replace,
expectErr: assert.Error,
},
{
name: "PatchList fails",
lrh: siteMock.NewListRestoreHandler(
nil,
nil,
errors.New("failed to patch list")),
collisionPolicy: control.Replace,
expectErr: assert.Error,
},
{
name: "PostList passes for stored list",
lrh: siteMock.NewListRestoreHandler(
nil,
nil,
nil),
collisionPolicy: control.Replace,
expectErr: assert.NoError,
expectCollisionCount: 1,
},
{
name: "Skip collison policy",
lrh: siteMock.NewListRestoreHandler(
nil,
nil,
nil),
collisionPolicy: control.Skip,
expectErr: assert.Error,
expectCollisionCount: 1,
},
{
name: "Copy collison policy",
lrh: siteMock.NewListRestoreHandler(
nil,
nil,
nil),
collisionPolicy: control.Copy,
expectErr: assert.NoError,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
mockData := generateListData(t, service, list)
cl := count.New()
restoreCfg.OnCollision = test.collisionPolicy
_, err := restoreListItem(
ctx,
test.lrh,
mockData,
suite.siteID,
restoreCfg,
collisionKeyToItemID,
cl,
fault.New(true))
test.expectErr(t, err)
if test.collisionPolicy == control.Skip {
assert.Equal(t, core.ErrAlreadyExists.Error(), err.Error())
}
if test.collisionPolicy == control.Copy {
assert.Zero(t, cl.Get(count.CollisionSkip))
assert.Zero(t, cl.Get(count.CollisionReplace))
}
assert.Equal(t, test.expectCollisionCount, cl.Get(policyToKey[test.collisionPolicy]))
})
}
}
func deleteList(
ctx context.Context,
t *testing.T,
@ -234,46 +407,40 @@ func deleteList(
}
}
func setupDependencies(
suite tester.Suite,
ac api.Client,
siteID string,
creds account.M365Config,
listTemplate string) (
string, listsRestoreHandler, string, *dataMock.Item,
) {
t := suite.T()
testName := "MockListing"
func generateListData(
t *testing.T,
service *graph.Service,
list models.Listable,
) *dataMock.Item {
listName := ptr.Val(list.GetDisplayName())
lrh := NewListsRestoreHandler(siteID, ac.Lists())
service := createTestService(t, creds)
listInfo := models.NewListInfo()
listInfo.SetTemplate(ptr.To(listTemplate))
listing := spMock.ListDefault("Mock List")
listing.SetDisplayName(&testName)
listing.SetList(listInfo)
byteArray, err := service.Serialize(listing)
byteArray, err := service.Serialize(list)
require.NoError(t, err, clues.ToCore(err))
destName := testdata.DefaultRestoreConfig("").Location
listData, err := data.NewPrefetchedItemWithInfo(
io.NopCloser(bytes.NewReader(byteArray)),
testName,
details.ItemInfo{SharePoint: api.ListToSPInfo(listing)})
listName,
details.ItemInfo{SharePoint: api.ListToSPInfo(list)})
require.NoError(t, err, clues.ToCore(err))
r, err := readers.NewVersionedRestoreReader(listData.ToReader())
require.NoError(t, err)
mockData := &dataMock.Item{
ItemID: testName,
ItemID: listName,
Reader: r,
}
return testName, lrh, destName, mockData
return mockData
}
func stubList(listTemplate, listDisplayName string) models.Listable {
listInfo := models.NewListInfo()
listInfo.SetTemplate(ptr.To(listTemplate))
listing := spMock.ListDefault("Mock List")
listing.SetDisplayName(ptr.To(listDisplayName))
listing.SetList(listInfo)
return listing
}

View File

@ -56,6 +56,7 @@ func (h *sharepointHandler) ConsumeRestoreCollections(
caches = drive.NewRestoreCaches(h.backupDriveIDNames)
el = errs.Local()
cl = ctr.Local()
)
// Reorder collections so that the parents directories are created
@ -77,6 +78,7 @@ func (h *sharepointHandler) ConsumeRestoreCollections(
"restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ProtectedResource()),
"full_path", dc.FullPath())
collisionKeyToItemID map[string]string
)
switch dc.FullPath().Category() {
@ -98,12 +100,20 @@ func (h *sharepointHandler) ConsumeRestoreCollections(
ctr)
case path.ListsCategory:
collisionKeyToItemID, err = listsRh.GetListsByCollisionKey(ictx)
if err != nil {
el.AddRecoverable(ictx, clues.Wrap(err, "building lists collision map"))
continue
}
metrics, err = site.RestoreListCollection(
ictx,
listsRh,
dc,
rcc.RestoreConfig.Location,
rcc.RestoreConfig,
deets,
collisionKeyToItemID,
cl,
errs)
case path.PagesCategory:

View File

@ -1,62 +0,0 @@
package sharepoint
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type SharepointRestoreUnitSuite struct {
tester.Suite
}
func TestSharepointRestoreUnitSuite(t *testing.T) {
suite.Run(t, &SharepointRestoreUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *SharepointRestoreUnitSuite) TestSharePointHandler_ConsumeRestoreCollections_noErrorOnLists() {
t := suite.T()
siteID := "site-id"
ctx, flush := tester.NewContext(t)
defer flush()
pr := idname.NewProvider(siteID, siteID)
rcc := inject.RestoreConsumerConfig{
ProtectedResource: pr,
}
pth, err := path.Builder{}.
Append("lists").
ToDataLayerPath(
"tenant",
siteID,
path.SharePointService,
path.ListsCategory,
false)
require.NoError(t, err, clues.ToCore(err))
dcs := []data.RestoreCollection{
mock.Collection{Path: pth},
}
sh := NewSharePointHandler(api.Client{}, nil)
_, _, err = sh.ConsumeRestoreCollections(
ctx,
rcc,
dcs,
fault.New(false),
nil)
require.NoError(t, err, "Sharepoint lists restore")
}

View File

@ -158,13 +158,13 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
ItemInfo: ItemInfo{
SharePoint: &SharePointInfo{
ItemType: SharePointList,
Created: now,
Modified: now,
WebURL: "https://example.com/Lists/list1",
List: &ListInfo{
Name: "list1",
ItemCount: 50,
Template: "genericList",
Created: now,
Modified: now,
WebURL: "https://10rqc2.sharepoint.com/sites/site-4754-small-lists/Lists/list1",
},
},
},
@ -188,13 +188,13 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
ItemInfo: ItemInfo{
SharePoint: &SharePointInfo{
ItemType: SharePointList,
Created: now,
Modified: now,
WebURL: "https://example.com/Lists/Shared%20Documents",
List: &ListInfo{
Name: "Shared%20Documents",
ItemCount: 50,
Template: "documentLibrary",
Created: now,
Modified: now,
WebURL: "https://10rqc2.sharepoint.com/sites/site-4754-small-lists/Lists/Shared%20Documents",
},
},
},

View File

@ -55,9 +55,6 @@ type ListInfo struct {
Name string `json:"name,omitempty"`
ItemCount int64 `json:"itemCount,omitempty"`
Template string `json:"template,omitempty"`
WebURL string `json:"webUrl,omitempty"`
Created time.Time `json:"created,omitempty"`
Modified time.Time `json:"modified,omitempty"`
}
// Headers returns the human-readable names of properties in a SharePointInfo
@ -91,8 +88,8 @@ func (i SharePointInfo) Values() []string {
return []string{
i.List.Name,
fmt.Sprintf("%d", i.List.ItemCount),
dttm.FormatToTabularDisplay(i.List.Created),
dttm.FormatToTabularDisplay(i.List.Modified),
dttm.FormatToTabularDisplay(i.Created),
dttm.FormatToTabularDisplay(i.Modified),
}
}

View File

@ -410,6 +410,46 @@ func (s *sharePoint) ModifiedBefore(timeStrings string) []SharePointScope {
}
}
func (s *sharePoint) ListModifiedAfter(timeStrings string) []SharePointScope {
return []SharePointScope{
makeInfoScope[SharePointScope](
SharePointListItem,
SharePointListInfoModifiedAfter,
[]string{timeStrings},
filters.Less),
}
}
func (s *sharePoint) ListModifiedBefore(timeStrings string) []SharePointScope {
return []SharePointScope{
makeInfoScope[SharePointScope](
SharePointListItem,
SharePointListInfoModifiedBefore,
[]string{timeStrings},
filters.Greater),
}
}
func (s *sharePoint) ListCreatedAfter(timeStrings string) []SharePointScope {
return []SharePointScope{
makeInfoScope[SharePointScope](
SharePointListItem,
SharePointListInfoCreatedAfter,
[]string{timeStrings},
filters.Less),
}
}
func (s *sharePoint) ListCreatedBefore(timeStrings string) []SharePointScope {
return []SharePointScope{
makeInfoScope[SharePointScope](
SharePointListItem,
SharePointListInfoCreatedBefore,
[]string{timeStrings},
filters.Greater),
}
}
// ---------------------------------------------------------------------------
// Categories
// ---------------------------------------------------------------------------
@ -440,6 +480,11 @@ const (
SharePointInfoModifiedAfter sharePointCategory = "SharePointInfoModifiedAfter"
SharePointInfoModifiedBefore sharePointCategory = "SharePointInfoModifiedBefore"
SharePointListInfoModifiedAfter sharePointCategory = "SharePointListInfoModifiedAfter"
SharePointListInfoModifiedBefore sharePointCategory = "SharePointListInfoModifiedBefore"
SharePointListInfoCreatedAfter sharePointCategory = "SharePointListInfoCreatedAfter"
SharePointListInfoCreatedBefore sharePointCategory = "SharePointListInfoCreatedBefore"
// library drive selection
SharePointInfoLibraryDrive sharePointCategory = "SharePointInfoLibraryDrive"
)
@ -479,7 +524,9 @@ func (c sharePointCategory) leafCat() categorizer {
SharePointInfoCreatedAfter, SharePointInfoCreatedBefore,
SharePointInfoModifiedAfter, SharePointInfoModifiedBefore:
return SharePointLibraryItem
case SharePointList, SharePointListItem:
case SharePointList, SharePointListItem,
SharePointListInfoModifiedAfter, SharePointListInfoModifiedBefore,
SharePointListInfoCreatedAfter, SharePointListInfoCreatedBefore:
return SharePointListItem
case SharePointPage, SharePointPageFolder:
return SharePointPage
@ -715,9 +762,11 @@ func (s SharePointScope) matchesInfo(dii details.ItemInfo) bool {
switch infoCat {
case SharePointWebURL:
i = info.WebURL
case SharePointInfoCreatedAfter, SharePointInfoCreatedBefore:
case SharePointInfoCreatedAfter, SharePointInfoCreatedBefore,
SharePointListInfoCreatedAfter, SharePointListInfoCreatedBefore:
i = dttm.Format(info.Created)
case SharePointInfoModifiedAfter, SharePointInfoModifiedBefore:
case SharePointInfoModifiedAfter, SharePointInfoModifiedBefore,
SharePointListInfoModifiedAfter, SharePointListInfoModifiedBefore:
i = dttm.Format(info.Modified)
case SharePointInfoLibraryDrive:
ds := []string{}

View File

@ -544,6 +544,19 @@ func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() {
{"file modified before future", host, sel.ModifiedBefore(dttm.Format(future)), assert.True},
{"file modified before now", host, sel.ModifiedBefore(dttm.Format(now)), assert.False},
{"file modified before epoch", host, sel.ModifiedBefore(dttm.Format(now)), assert.False},
{"list create after the epoch", host, sel.ListCreatedAfter(dttm.Format(epoch)), assert.True},
{"list create after now", host, sel.ListCreatedAfter(dttm.Format(now)), assert.False},
{"list create after later", url, sel.ListCreatedAfter(dttm.Format(future)), assert.False},
{"list create before future", host, sel.ListCreatedBefore(dttm.Format(future)), assert.True},
{"list create before now", host, sel.ListCreatedBefore(dttm.Format(now)), assert.False},
{"list create before modification", host, sel.ListCreatedBefore(dttm.Format(modification)), assert.True},
{"list create before epoch", host, sel.ListCreatedBefore(dttm.Format(now)), assert.False},
{"list modified after the epoch", host, sel.ListModifiedAfter(dttm.Format(epoch)), assert.True},
{"list modified after now", host, sel.ListModifiedAfter(dttm.Format(now)), assert.True},
{"list modified after later", host, sel.ListModifiedAfter(dttm.Format(future)), assert.False},
{"list modified before future", host, sel.ListModifiedBefore(dttm.Format(future)), assert.True},
{"list modified before now", host, sel.ListModifiedBefore(dttm.Format(now)), assert.False},
{"list modified before epoch", host, sel.ListModifiedBefore(dttm.Format(now)), assert.False},
{"in library", host, sel.Library("included-library"), assert.True},
{"not in library", host, sel.Library("not-included-library"), assert.False},
{"library id", host, sel.Library("1234"), assert.True},

View File

@ -244,6 +244,22 @@ func (c Lists) DeleteList(
return graph.Wrap(ctx, err, "deleting list").OrNil()
}
func (c Lists) PatchList(
ctx context.Context,
siteID, listID string,
list models.Listable,
) (models.Listable, error) {
patchedList, err := c.Stable.
Client().
Sites().
BySiteId(siteID).
Lists().
ByListId(listID).
Patch(ctx, list, nil)
return patchedList, graph.Wrap(ctx, err, "patching list").OrNil()
}
func BytesToListable(bytes []byte) (models.Listable, error) {
parsable, err := CreateFromBytes(bytes, models.CreateListFromDiscriminatorValue)
if err != nil {
@ -591,13 +607,12 @@ func ListToSPInfo(lst models.Listable) *details.SharePointInfo {
return &details.SharePointInfo{
ItemType: details.SharePointList,
Modified: modified,
Created: created,
WebURL: webURL,
List: &details.ListInfo{
Name: name,
ItemCount: int64(count),
Template: template,
Created: created,
Modified: modified,
WebURL: webURL,
},
}
}

View File

@ -774,6 +774,7 @@ func (suite *ListsAPIIntgSuite) TestLists_GetListByID() {
list.SetId(ptr.To(listID))
list.SetDisplayName(ptr.To(listName))
list.SetList(listInfo)
list.SetCreatedDateTime(ptr.To(time.Now()))
list.SetLastModifiedDateTime(ptr.To(time.Now()))
txtColumnDef := models.NewColumnDefinition()
@ -911,8 +912,8 @@ func (suite *ListsAPIIntgSuite) TestLists_GetListByID() {
assert.Equal(t, listName, info.List.Name)
assert.Equal(t, int64(1), info.List.ItemCount)
assert.Equal(t, listTemplate, info.List.Template)
assert.NotEmpty(t, info.List.Modified)
assert.NotEmpty(t, info.Modified)
assert.NotEmpty(t, info.Created)
})
}
}
@ -1016,6 +1017,50 @@ func (suite *ListsAPIIntgSuite) TestLists_PostList_invalidTemplate() {
}
}
func (suite *ListsAPIIntgSuite) TestLists_PatchList() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
var (
acl = suite.its.ac.Lists()
siteID = suite.its.site.id
listName = "old-list-name"
newListName = "new-list-name"
)
fieldsData, list := getFieldsDataAndList()
createdList, err := acl.PostList(ctx, siteID, listName, list, fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, listName, ptr.Val(createdList.GetDisplayName()))
listID := ptr.Val(createdList.GetId())
newList := models.NewList()
newList.SetDisplayName(ptr.To(newListName))
patchedList, err := acl.PatchList(ctx, siteID, listID, newList)
require.NoError(t, err)
assert.Equal(t, newListName, ptr.Val(patchedList.GetDisplayName()))
patchedList, _, err = acl.GetListByID(ctx, siteID, listID)
require.NoError(t, err)
newListItems := patchedList.GetItems()
require.Less(t, 0, len(newListItems))
newListItemFields := newListItems[0].GetFields()
require.NotEmpty(t, newListItemFields)
newListItemsData := newListItemFields.GetAdditionalData()
require.NotEmpty(t, newListItemsData)
assert.Equal(t, fieldsData["itemName"], newListItemsData["itemName"])
err = acl.DeleteList(ctx, siteID, ptr.Val(patchedList.GetId()))
require.NoError(t, err)
}
func (suite *ListsAPIIntgSuite) TestLists_DeleteList() {
t := suite.T()