From 1c520db3bca7ae53b80fb1c397f66da9535e42c1 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:17:11 -0700 Subject: [PATCH] Use generic item in SharePoint lists/pages (#4367) #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #4191 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- .../m365/collection/site/collection.go | 70 +++---------------- .../m365/collection/site/collection_test.go | 49 +++++-------- .../m365/service/sharepoint/api/pages_test.go | 8 ++- 3 files changed, 30 insertions(+), 97 deletions(-) diff --git a/src/internal/m365/collection/site/collection.go b/src/internal/m365/collection/site/collection.go index 95d77acb2..422ed4b2a 100644 --- a/src/internal/m365/collection/site/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "io" - "time" "github.com/alcionai/clues" "github.com/microsoft/kiota-abstractions-go/serialization" @@ -40,12 +39,7 @@ const ( Pages DataCategory = 2 ) -var ( - _ data.BackupCollection = &Collection{} - _ data.Item = &Item{} - _ data.ItemInfo = &Item{} - _ data.ItemModTime = &Item{} -) +var _ data.BackupCollection = &Collection{} // Collection is the SharePoint.List implementation of data.Collection. SharePoint.Libraries collections are supported // by the oneDrive.Collection as the calls are identical for populating the Collection @@ -120,43 +114,6 @@ func (sc *Collection) Items( return sc.data } -type Item struct { - id string - data io.ReadCloser - info *details.SharePointInfo - modTime time.Time - - // true if the item was marked by graph as deleted. - deleted bool -} - -func NewItem(name string, d io.ReadCloser) *Item { - return &Item{ - id: name, - data: d, - } -} - -func (sd *Item) ID() string { - return sd.id -} - -func (sd *Item) ToReader() io.ReadCloser { - return sd.data -} - -func (sd Item) Deleted() bool { - return sd.deleted -} - -func (sd *Item) Info() (details.ItemInfo, error) { - return details.ItemInfo{SharePoint: sd.info}, nil -} - -func (sd *Item) ModTime() time.Time { - return sd.modTime -} - func (sc *Collection) finishPopulation( ctx context.Context, metrics support.CollectionMetrics, @@ -251,20 +208,13 @@ func (sc *Collection) retrieveLists( size := int64(len(byteArray)) if size > 0 { - t := time.Now() - if t1 := lst.GetLastModifiedDateTime(); t1 != nil { - t = *t1 - } - metrics.Bytes += size metrics.Successes++ - sc.data <- &Item{ - id: ptr.Val(lst.GetId()), - data: io.NopCloser(bytes.NewReader(byteArray)), - info: ListToSPInfo(lst, size), - modTime: t, - } + sc.data <- data.NewPrefetchedItem( + io.NopCloser(bytes.NewReader(byteArray)), + ptr.Val(lst.GetId()), + details.ItemInfo{SharePoint: ListToSPInfo(lst, size)}) progress <- struct{}{} } @@ -322,12 +272,10 @@ func (sc *Collection) retrievePages( if size > 0 { metrics.Bytes += size metrics.Successes++ - sc.data <- &Item{ - id: ptr.Val(pg.GetId()), - data: io.NopCloser(bytes.NewReader(byteArray)), - info: pageToSPInfo(pg, root, size), - modTime: ptr.OrNow(pg.GetLastModifiedDateTime()), - } + sc.data <- data.NewPrefetchedItem( + io.NopCloser(bytes.NewReader(byteArray)), + ptr.Val(pg.GetId()), + details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)}) progress <- struct{}{} } diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index 0be5c2dc8..3d0336217 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" @@ -58,21 +59,6 @@ func TestSharePointCollectionSuite(t *testing.T) { }) } -func (suite *SharePointCollectionSuite) TestCollection_Item_Read() { - t := suite.T() - m := []byte("test message") - name := "aFile" - sc := &Item{ - id: name, - data: io.NopCloser(bytes.NewReader(m)), - } - readData, err := io.ReadAll(sc.ToReader()) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, name, sc.id) - assert.Equal(t, readData, m) -} - // TestListCollection tests basic functionality to create // SharePoint collection and to use the data stream channel. func (suite *SharePointCollectionSuite) TestCollection_Items() { @@ -88,7 +74,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { name, itemName string scope selectors.SharePointScope getDir func(t *testing.T) path.Path - getItem func(t *testing.T, itemName string) *Item + getItem func(t *testing.T, itemName string) data.Item }{ { name: "List", @@ -106,7 +92,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { return dir }, - getItem: func(t *testing.T, name string) *Item { + getItem: func(t *testing.T, name string) data.Item { ow := kioser.NewJsonSerializationWriter() listing := spMock.ListDefault(name) listing.SetDisplayName(&name) @@ -117,11 +103,10 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { byteArray, err := ow.GetSerializedContent() require.NoError(t, err, clues.ToCore(err)) - data := &Item{ - id: name, - data: io.NopCloser(bytes.NewReader(byteArray)), - info: ListToSPInfo(listing, int64(len(byteArray))), - } + data := data.NewPrefetchedItem( + io.NopCloser(bytes.NewReader(byteArray)), + name, + details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))}) return data }, @@ -142,16 +127,15 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { return dir }, - getItem: func(t *testing.T, itemName string) *Item { + getItem: func(t *testing.T, itemName string) data.Item { byteArray := spMock.Page(itemName) page, err := betaAPI.CreatePageFromBytes(byteArray) require.NoError(t, err, clues.ToCore(err)) - data := &Item{ - id: itemName, - data: io.NopCloser(bytes.NewReader(byteArray)), - info: betaAPI.PageInfo(page, int64(len(byteArray))), - } + data := data.NewPrefetchedItem( + io.NopCloser(bytes.NewReader(byteArray)), + itemName, + details.ItemInfo{SharePoint: betaAPI.PageInfo(page, int64(len(byteArray)))}) return data }, @@ -210,11 +194,10 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { byteArray, err := service.Serialize(listing) require.NoError(t, err, clues.ToCore(err)) - listData := &Item{ - id: testName, - data: io.NopCloser(bytes.NewReader(byteArray)), - info: ListToSPInfo(listing, int64(len(byteArray))), - } + listData := data.NewPrefetchedItem( + io.NopCloser(bytes.NewReader(byteArray)), + testName, + details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))}) destName := testdata.DefaultRestoreConfig("").Location diff --git a/src/internal/m365/service/sharepoint/api/pages_test.go b/src/internal/m365/service/sharepoint/api/pages_test.go index a834f10ea..f462805d2 100644 --- a/src/internal/m365/service/sharepoint/api/pages_test.go +++ b/src/internal/m365/service/sharepoint/api/pages_test.go @@ -4,13 +4,14 @@ import ( "bytes" "io" "testing" + "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/collection/site" + "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" @@ -108,9 +109,10 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() { //nolint:lll byteArray := spMock.Page("Byte Test") - pageData := site.NewItem( + pageData := data.NewUnindexedPrefetchedItem( + io.NopCloser(bytes.NewReader(byteArray)), testName, - io.NopCloser(bytes.NewReader(byteArray))) + time.Now()) info, err := api.RestoreSitePage( ctx,