From 66664b69e58d9022080d19c61084aa211055e5c4 Mon Sep 17 00:00:00 2001 From: Hitesh Pattanayak <48874082+HiteshRepo@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:29:10 +0530 Subject: [PATCH] skips backup of non-recoverable list templates (#4940) Skips backup of non-recoverable list templates. #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Issue(s) #4754 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/m365/collection/site/backup.go | 6 ++- .../m365/collection/site/collection.go | 8 +-- .../m365/collection/site/collection_test.go | 51 ------------------- src/internal/m365/collection/site/restore.go | 2 +- .../m365/collection/site/restore_test.go | 2 +- src/pkg/backup/details/details_test.go | 28 ---------- src/pkg/backup/details/iteminfo.go | 3 +- src/pkg/backup/details/model.go | 6 +-- src/pkg/services/m365/api/lists.go | 10 ++-- src/pkg/services/m365/api/lists_test.go | 2 +- 10 files changed, 19 insertions(+), 99 deletions(-) diff --git a/src/internal/m365/collection/site/backup.go b/src/internal/m365/collection/site/backup.go index 22636f2f7..3c6a61c27 100644 --- a/src/internal/m365/collection/site/backup.go +++ b/src/internal/m365/collection/site/backup.go @@ -154,7 +154,7 @@ func CollectLists( var ( el = errs.Local() spcs = make([]data.BackupCollection, 0) - acc = api.CallConfig{Select: idAnd("displayName")} + acc = api.CallConfig{Select: idAnd("list")} ) lists, err := bh.GetItems(ctx, acc) @@ -167,6 +167,10 @@ func CollectLists( break } + if api.SkipListTemplates.HasKey(ptr.Val(list.GetList().GetTemplate())) { + continue + } + dir, err := path.Build( tenantID, bpc.ProtectedResource.ID(), diff --git a/src/internal/m365/collection/site/collection.go b/src/internal/m365/collection/site/collection.go index 491360060..505028048 100644 --- a/src/internal/m365/collection/site/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -350,15 +350,9 @@ func (sc *Collection) handleListItems( metrics.Bytes += size metrics.Successes++ - template := "" - if list != nil && list.GetList() != nil { - template = ptr.Val(list.GetList().GetTemplate()) - } - rc := io.NopCloser(bytes.NewReader(entryBytes)) itemInfo := details.ItemInfo{ - SharePoint: info, - NotRecoverable: template == api.WebTemplateExtensionsListTemplateName, + SharePoint: info, } item, err := data.NewPrefetchedItemWithInfo(rc, listID, itemInfo) diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index 56c6b0057..547503a6a 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -7,12 +7,10 @@ import ( "github.com/alcionai/clues" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/site/mock" betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" @@ -76,7 +74,6 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { tables := []struct { name, itemName string - notRecoverable bool scope selectors.SharePointScope getter getItemByIDer getDir func(t *testing.T) path.Path @@ -123,53 +120,6 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { return data }, }, - { - name: "List with wte template", - itemName: "MockListing", - notRecoverable: true, - scope: sel.Lists(selectors.Any())[0], - getter: &mock.ListHandler{}, - getDir: func(t *testing.T) path.Path { - dir, err := path.Build( - tenant, - user, - path.SharePointService, - path.ListsCategory, - false, - dirRoot) - require.NoError(t, err, clues.ToCore(err)) - - return dir - }, - getItem: func(t *testing.T, name string) data.Item { - ow := kioser.NewJsonSerializationWriter() - - listInfo := models.NewListInfo() - listInfo.SetTemplate(ptr.To("webTemplateExtensionsList")) - - listing := spMock.ListDefault(name) - listing.SetDisplayName(&name) - listing.SetList(listInfo) - - err := ow.WriteObjectValue("", listing) - require.NoError(t, err, clues.ToCore(err)) - - byteArray, err := ow.GetSerializedContent() - require.NoError(t, err, clues.ToCore(err)) - - info := &details.SharePointInfo{ - ItemName: name, - } - - data, err := data.NewPrefetchedItemWithInfo( - io.NopCloser(bytes.NewReader(byteArray)), - name, - details.ItemInfo{SharePoint: info, NotRecoverable: true}) - require.NoError(t, err, clues.ToCore(err)) - - return data - }, - }, { name: "Pages", itemName: "MockPages", @@ -236,7 +186,6 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { assert.NotNil(t, info) assert.NotNil(t, info.SharePoint) assert.Equal(t, test.itemName, info.SharePoint.ItemName) - assert.Equal(t, test.notRecoverable, info.NotRecoverable) }) } } diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index 334dd486d..066991c95 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -211,7 +211,7 @@ func RestoreListCollection( restoreContainerName, errs) if err != nil && - errors.Is(err, api.ErrCannotCreateWebTemplateExtension) { + errors.Is(err, api.ErrSkippableListTemplate) { continue } diff --git a/src/internal/m365/collection/site/restore_test.go b/src/internal/m365/collection/site/restore_test.go index ebbfc9c4f..6cae7f508 100644 --- a/src/internal/m365/collection/site/restore_test.go +++ b/src/internal/m365/collection/site/restore_test.go @@ -105,7 +105,7 @@ func (suite *SharePointRestoreSuite) TestListCollection_Restore_invalidListTempl _, err := restoreListItem(ctx, lrh, mockData, suite.siteID, destName, fault.New(true)) require.Error(t, err) - assert.Contains(t, err.Error(), api.ErrCannotCreateWebTemplateExtension.Error()) + assert.Contains(t, err.Error(), api.ErrSkippableListTemplate.Error()) } func deleteList( diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index 8ba7b94dd..a0ba8d077 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -907,34 +907,6 @@ var pathItemsTable = []struct { expectRepoRefs: []string{"abcde", "12345", "foo.meta"}, expectLocationRefs: []string{"locationref", "locationref2", "locationref.dirmeta"}, }, - { - name: "multiple entries with not recoverables", - ents: []Entry{ - { - RepoRef: "abcde", - LocationRef: "locationref", - }, - { - RepoRef: "foo.meta", - LocationRef: "locationref.dirmeta", - ItemRef: "itemref.meta", - ItemInfo: ItemInfo{ - SharePoint: &SharePointInfo{ItemType: SharePointList}, - }, - }, - { - RepoRef: "invalid-template-list-file", - LocationRef: "locationref-invalid-template-list-file", - ItemRef: "itemref-template-list-file", - ItemInfo: ItemInfo{ - SharePoint: &SharePointInfo{ItemType: SharePointList}, - NotRecoverable: true, - }, - }, - }, - expectRepoRefs: []string{"abcde", "foo.meta"}, - expectLocationRefs: []string{"locationref", "locationref.dirmeta"}, - }, } func (suite *DetailsUnitSuite) TestDetailsModel_Path() { diff --git a/src/pkg/backup/details/iteminfo.go b/src/pkg/backup/details/iteminfo.go index 3e316a911..e2eaf357e 100644 --- a/src/pkg/backup/details/iteminfo.go +++ b/src/pkg/backup/details/iteminfo.go @@ -74,8 +74,7 @@ type ItemInfo struct { OneDrive *OneDriveInfo `json:"oneDrive,omitempty"` Groups *GroupsInfo `json:"groups,omitempty"` // Optional item extension data - Extension *ExtensionData `json:"extension,omitempty"` - NotRecoverable bool `json:"notRecoverable,omitempty"` + Extension *ExtensionData `json:"extension,omitempty"` } // typedInfo should get embedded in each sesrvice type to track diff --git a/src/pkg/backup/details/model.go b/src/pkg/backup/details/model.go index dcd75ce56..9687cc731 100644 --- a/src/pkg/backup/details/model.go +++ b/src/pkg/backup/details/model.go @@ -68,8 +68,7 @@ func (dm DetailsModel) Paths() []string { for _, ent := range dm.Entries { if ent.Folder != nil || - ent.isMetaFile() || - ent.NotRecoverable { + ent.isMetaFile() { continue } @@ -88,8 +87,7 @@ func (dm DetailsModel) Items() entrySet { for i := 0; i < len(dm.Entries); i++ { ent := dm.Entries[i] if ent.Folder != nil || - ent.isMetaFile() || - ent.NotRecoverable { + ent.isMetaFile() { continue } diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go index 4d79e01c6..8aad0fa9b 100644 --- a/src/pkg/services/m365/api/lists.go +++ b/src/pkg/services/m365/api/lists.go @@ -14,7 +14,7 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -var ErrCannotCreateWebTemplateExtension = clues.New("unable to create webTemplateExtension type lists") +var ErrSkippableListTemplate = clues.New("unable to create lists with skippable templates") const ( AttachmentsColumnName = "Attachments" @@ -83,6 +83,10 @@ var readOnlyFieldNames = keys.Set{ ModifiedColumnName: {}, } +var SkipListTemplates = keys.Set{ + WebTemplateExtensionsListTemplateName: {}, +} + // --------------------------------------------------------------------------- // controller // --------------------------------------------------------------------------- @@ -258,8 +262,8 @@ func (c Lists) PostList( if newList != nil && newList.GetList() != nil && - ptr.Val(newList.GetList().GetTemplate()) == WebTemplateExtensionsListTemplateName { - return nil, clues.StackWC(ctx, ErrCannotCreateWebTemplateExtension) + SkipListTemplates.HasKey(ptr.Val(newList.GetList().GetTemplate())) { + return nil, clues.StackWC(ctx, ErrSkippableListTemplate) } // Restore to List base to M365 back store diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go index b55cfc808..631f392bf 100644 --- a/src/pkg/services/m365/api/lists_test.go +++ b/src/pkg/services/m365/api/lists_test.go @@ -788,7 +788,7 @@ func (suite *ListsAPIIntgSuite) TestLists_PostList_invalidTemplate() { _, err = acl.PostList(ctx, siteID, listName, oldListByteArray, fault.New(true)) require.Error(t, err) - assert.Equal(t, ErrCannotCreateWebTemplateExtension.Error(), err.Error()) + assert.Equal(t, ErrSkippableListTemplate.Error(), err.Error()) } func (suite *ListsAPIIntgSuite) TestLists_DeleteList() {