From 1d971fb2ef3f9609bad933d7f03947585765293f Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Fri, 1 Dec 2023 22:17:24 -0800 Subject: [PATCH] Use custom drive items during backup --- .../m365/collection/drive/collection.go | 29 ++++++++++-------- .../m365/collection/drive/collection_test.go | 5 ++-- .../m365/collection/drive/collections.go | 5 +++- .../m365/collection/drive/collections_tree.go | 3 +- .../m365/collection/drive/group_handler.go | 4 +-- .../m365/collection/drive/handler_utils.go | 7 ++--- .../m365/collection/drive/handlers.go | 3 +- src/internal/m365/collection/drive/item.go | 16 ++-------- .../m365/collection/drive/item_test.go | 7 +++-- src/internal/m365/collection/drive/restore.go | 3 +- .../m365/collection/drive/site_handler.go | 3 +- .../collection/drive/user_drive_handler.go | 3 +- .../m365/service/onedrive/mock/handlers.go | 5 ++-- src/pkg/services/m365/api/graph/errors.go | 30 ++++++++++--------- .../services/m365/api/graph/errors_test.go | 3 +- src/pkg/services/m365/custom/drive_item.go | 11 +++++++ 16 files changed, 78 insertions(+), 59 deletions(-) diff --git a/src/internal/m365/collection/drive/collection.go b/src/internal/m365/collection/drive/collection.go index 6375a71b1..16f0ff065 100644 --- a/src/internal/m365/collection/drive/collection.go +++ b/src/internal/m365/collection/drive/collection.go @@ -28,6 +28,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) const ( @@ -52,7 +53,7 @@ type Collection struct { // represents folderPath path.Path // M365 IDs of file items within this collection - driveItems map[string]models.DriveItemable + driveItems map[string]custom.LiteDriveItemable // Primary M365 ID of the drive this collection was created from driveID string @@ -172,7 +173,7 @@ func newColl( protectedResource: resource, folderPath: currPath, prevPath: prevPath, - driveItems: map[string]models.DriveItemable{}, + driveItems: map[string]custom.LiteDriveItemable{}, driveID: driveID, data: dataCh, statusUpdater: statusUpdater, @@ -191,8 +192,10 @@ func newColl( // populated. The return values denotes if the item was previously // present or is new one. func (oc *Collection) Add(item models.DriveItemable) bool { - _, found := oc.driveItems[ptr.Val(item.GetId())] - oc.driveItems[ptr.Val(item.GetId())] = item + liteItem := custom.ToLiteDriveItemable(item) + + _, found := oc.driveItems[ptr.Val(liteItem.GetId())] + oc.driveItems[ptr.Val(liteItem.GetId())] = liteItem // if !found, it's a new addition return !found @@ -277,7 +280,7 @@ func (oc Collection) DoNotMergeItems() bool { func (oc *Collection) getDriveItemContent( ctx context.Context, driveID string, - item models.DriveItemable, + item custom.LiteDriveItemable, errs *fault.Bus, ) (io.ReadCloser, error) { var ( @@ -360,7 +363,7 @@ func downloadContent( ctx context.Context, iaag itemAndAPIGetter, uc getItemPropertyer, - item models.DriveItemable, + item custom.LiteDriveItemable, driveID string, counter *count.Bus, ) (io.ReadCloser, error) { @@ -395,7 +398,9 @@ func downloadContent( return nil, clues.Wrap(err, "retrieving expired item") } - content, err = downloadItem(ctx, iaag, di) + ldi := custom.ToLiteDriveItemable(di) + + content, err = downloadItem(ctx, iaag, ldi) if err != nil { return nil, clues.Wrap(err, "content download retry") } @@ -489,7 +494,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) { wg.Add(1) - go func(item models.DriveItemable) { + go func(item custom.LiteDriveItemable) { defer wg.Done() defer func() { <-semaphoreCh }() @@ -513,14 +518,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) { type lazyItemGetter struct { info *details.ItemInfo - item models.DriveItemable + item custom.LiteDriveItemable driveID string suffix string itemExtensionFactory []extensions.CreateItemExtensioner contentGetter func( ctx context.Context, driveID string, - item models.DriveItemable, + item custom.LiteDriveItemable, errs *fault.Bus) (io.ReadCloser, error) } @@ -561,7 +566,7 @@ func (lig *lazyItemGetter) GetData( func (oc *Collection) streamDriveItem( ctx context.Context, parentPath *path.Builder, - item models.DriveItemable, + item custom.LiteDriveItemable, stats *driveStats, itemExtensionFactory []extensions.CreateItemExtensioner, errs *fault.Bus, @@ -584,7 +589,7 @@ func (oc *Collection) streamDriveItem( "item_name", clues.Hide(itemName), "item_size", itemSize) - item.SetParentReference(setName(item.GetParentReference(), oc.driveName)) + item.SetParentReference(custom.SetParentName(item.GetParentReference(), oc.driveName)) isFile := item.GetFile() != nil diff --git a/src/internal/m365/collection/drive/collection_test.go b/src/internal/m365/collection/drive/collection_test.go index 6ee30143a..8d66e5145 100644 --- a/src/internal/m365/collection/drive/collection_test.go +++ b/src/internal/m365/collection/drive/collection_test.go @@ -34,6 +34,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) // --------------------------------------------------------------------------- @@ -641,7 +642,7 @@ func (suite *GetDriveItemUnitTestSuite) TestGetDriveItem_error() { col.handler = mbh - _, err := col.getDriveItemContent(ctx, "driveID", stubItem, errs) + _, err := col.getDriveItemContent(ctx, "driveID", custom.ToLiteDriveItemable(stubItem), errs) if test.err == nil { assert.NoError(t, err, clues.ToCore(err)) return @@ -819,7 +820,7 @@ func (suite *GetDriveItemUnitTestSuite) TestDownloadContent() { mbh.GetResps = resps mbh.GetErrs = test.getErr - r, err := downloadContent(ctx, mbh, test.muc, item, driveID, count.New()) + r, err := downloadContent(ctx, mbh, test.muc, custom.ToLiteDriveItemable(item), driveID, count.New()) test.expect(t, r) test.expectErr(t, err, clues.ToCore(err)) }) diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index 1c199025e..704f9593b 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -27,6 +27,7 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) const ( @@ -955,7 +956,9 @@ func (c *Collections) processItem( "item_is_folder", isFolder) if item.GetMalware() != nil { - addtl := graph.ItemInfo(item) + // TODO(pandeyabs): Fix this after we move conversion logic to the top of this + // func. + addtl := graph.ItemInfo(custom.ToLiteDriveItemable(item)) skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) if isFolder { diff --git a/src/internal/m365/collection/drive/collections_tree.go b/src/internal/m365/collection/drive/collections_tree.go index d11841be8..752ccc9fd 100644 --- a/src/internal/m365/collection/drive/collections_tree.go +++ b/src/internal/m365/collection/drive/collections_tree.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) // --------------------------------------------------------------------------- @@ -464,7 +465,7 @@ func (c *Collections) addFolderToTree( driveID, folderID, folderName, - graph.ItemInfo(folder)) + graph.ItemInfo(custom.ToLiteDriveItemable(folder))) logger.Ctx(ctx).Infow("malware detected") diff --git a/src/internal/m365/collection/drive/group_handler.go b/src/internal/m365/collection/drive/group_handler.go index a603ce748..48f72f7e4 100644 --- a/src/internal/m365/collection/drive/group_handler.go +++ b/src/internal/m365/collection/drive/group_handler.go @@ -2,7 +2,6 @@ package drive import ( "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" @@ -11,6 +10,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) var _ BackupHandler = &groupBackupHandler{} @@ -105,7 +105,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) { func (h groupBackupHandler) AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item custom.LiteDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo { diff --git a/src/internal/m365/collection/drive/handler_utils.go b/src/internal/m365/collection/drive/handler_utils.go index dd1e415b4..8fb62490a 100644 --- a/src/internal/m365/collection/drive/handler_utils.go +++ b/src/internal/m365/collection/drive/handler_utils.go @@ -3,12 +3,11 @@ package drive import ( "strings" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) -func getItemCreator(item models.DriveItemable) string { +func getItemCreator(item custom.LiteDriveItemable) string { if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil { return "" } @@ -30,7 +29,7 @@ func getItemCreator(item models.DriveItemable) string { return *ed.(*string) } -func getItemDriveInfo(item models.DriveItemable) (string, string) { +func getItemDriveInfo(item custom.LiteDriveItemable) (string, string) { if item.GetParentReference() == nil { return "", "" } diff --git a/src/internal/m365/collection/drive/handlers.go b/src/internal/m365/collection/drive/handlers.go index 76c147144..be8bc04b5 100644 --- a/src/internal/m365/collection/drive/handlers.go +++ b/src/internal/m365/collection/drive/handlers.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) type ItemInfoAugmenter interface { @@ -23,7 +24,7 @@ type ItemInfoAugmenter interface { AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item custom.LiteDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo diff --git a/src/internal/m365/collection/drive/item.go b/src/internal/m365/collection/drive/item.go index 9fad7cd8e..2329a8cce 100644 --- a/src/internal/m365/collection/drive/item.go +++ b/src/internal/m365/collection/drive/item.go @@ -7,7 +7,6 @@ import ( "io" "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/ptr" @@ -18,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) const ( @@ -34,7 +34,7 @@ var downloadURLKeys = []string{ func downloadItem( ctx context.Context, ag api.Getter, - item models.DriveItemable, + item custom.LiteDriveItemable, ) (io.ReadCloser, error) { if item == nil { return nil, clues.New("nil item") @@ -152,7 +152,7 @@ func downloadItemMeta( ctx context.Context, getter GetItemPermissioner, driveID string, - item models.DriveItemable, + item custom.LiteDriveItemable, ) (io.ReadCloser, int, error) { meta := metadata.Metadata{ FileName: ptr.Val(item.GetName()), @@ -204,13 +204,3 @@ func driveItemWriter( return iw, ptr.Val(icu.GetUploadUrl()), nil } - -func setName(orig models.ItemReferenceable, driveName string) models.ItemReferenceable { - if orig == nil { - return nil - } - - orig.SetName(&driveName) - - return orig -} diff --git a/src/internal/m365/collection/drive/item_test.go b/src/internal/m365/collection/drive/item_test.go index 5be66f7c1..7707f1036 100644 --- a/src/internal/m365/collection/drive/item_test.go +++ b/src/internal/m365/collection/drive/item_test.go @@ -25,6 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) type ItemIntegrationSuite struct { @@ -123,7 +124,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { } // Read data for the file - itemData, err := downloadItem(ctx, bh, driveItem) + itemData, err := downloadItem(ctx, bh, custom.ToLiteDriveItemable(driveItem)) require.NoError(t, err, clues.ToCore(err)) size, err := io.Copy(io.Discard, itemData) @@ -462,7 +463,7 @@ func (suite *ItemUnitTestSuite) TestDownloadItem() { mg := mockGetter{ GetFunc: test.GetFunc, } - rc, err := downloadItem(ctx, mg, test.itemFunc()) + rc, err := downloadItem(ctx, mg, custom.ToLiteDriveItemable(test.itemFunc())) test.errorExpected(t, err, clues.ToCore(err)) test.rcExpected(t, rc) }) @@ -521,7 +522,7 @@ func (suite *ItemUnitTestSuite) TestDownloadItem_ConnectionResetErrorOnFirstRead mg := mockGetter{ GetFunc: GetFunc, } - rc, err := downloadItem(ctx, mg, itemFunc()) + rc, err := downloadItem(ctx, mg, custom.ToLiteDriveItemable(itemFunc())) errorExpected(t, err, clues.ToCore(err)) rcExpected(t, rc) diff --git a/src/internal/m365/collection/drive/restore.go b/src/internal/m365/collection/drive/restore.go index c842baa18..d1f44dea3 100644 --- a/src/internal/m365/collection/drive/restore.go +++ b/src/internal/m365/collection/drive/restore.go @@ -30,6 +30,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) const ( @@ -863,7 +864,7 @@ func restoreFile( dii := ir.AugmentItemInfo( details.ItemInfo{}, rcc.ProtectedResource, - newItem, + custom.ToLiteDriveItemable(newItem), written, nil) diff --git a/src/internal/m365/collection/drive/site_handler.go b/src/internal/m365/collection/drive/site_handler.go index 2500b565a..7eaa3583a 100644 --- a/src/internal/m365/collection/drive/site_handler.go +++ b/src/internal/m365/collection/drive/site_handler.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) type baseSiteHandler struct { @@ -33,7 +34,7 @@ func (h baseSiteHandler) NewDrivePager( func (h baseSiteHandler) AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item custom.LiteDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo { diff --git a/src/internal/m365/collection/drive/user_drive_handler.go b/src/internal/m365/collection/drive/user_drive_handler.go index dad4a1f6c..4b7a71366 100644 --- a/src/internal/m365/collection/drive/user_drive_handler.go +++ b/src/internal/m365/collection/drive/user_drive_handler.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) // --------------------------------------------------------------------------- @@ -42,7 +43,7 @@ func (h baseUserDriveHandler) NewDrivePager( func (h baseUserDriveHandler) AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item custom.LiteDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo { diff --git a/src/internal/m365/service/onedrive/mock/handlers.go b/src/internal/m365/service/onedrive/mock/handlers.go index f46011800..c028b2a75 100644 --- a/src/internal/m365/service/onedrive/mock/handlers.go +++ b/src/internal/m365/service/onedrive/mock/handlers.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api" apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) // --------------------------------------------------------------------------- @@ -165,7 +166,7 @@ func (h BackupHandler[T]) NewLocationIDer(driveID string, elems ...string) detai func (h BackupHandler[T]) AugmentItemInfo( details.ItemInfo, idname.Provider, - models.DriveItemable, + custom.LiteDriveItemable, int64, *path.Builder, ) details.ItemInfo { @@ -405,7 +406,7 @@ func (h RestoreHandler) NewDrivePager(string, []string) pagers.NonDeltaHandler[m func (h *RestoreHandler) AugmentItemInfo( details.ItemInfo, idname.Provider, - models.DriveItemable, + custom.LiteDriveItemable, int64, *path.Builder, ) details.ItemInfo { diff --git a/src/pkg/services/m365/api/graph/errors.go b/src/pkg/services/m365/api/graph/errors.go index 71bebe134..0983e4a79 100644 --- a/src/pkg/services/m365/api/graph/errors.go +++ b/src/pkg/services/m365/api/graph/errors.go @@ -10,7 +10,6 @@ import ( "syscall" "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" "github.com/pkg/errors" @@ -20,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) // --------------------------------------------------------------------------- @@ -510,18 +510,20 @@ func appendIf(a []any, k string, v *string) []any { // ItemInfo gathers potentially useful information about a drive item, // and aggregates that data into a map. -func ItemInfo(item models.DriveItemable) map[string]any { +func ItemInfo(item custom.LiteDriveItemable) map[string]any { m := map[string]any{} - creator := item.GetCreatedByUser() - if creator != nil { - m[fault.AddtlCreatedBy] = ptr.Val(creator.GetId()) - } + // TODO(pandeyabs): These fields are not available in the LiteDriveItemable + // yet. We need to add them. + // creator := item.GetCreatedByUser() + // if creator != nil { + // m[fault.AddtlCreatedBy] = ptr.Val(creator.GetId()) + // } - lastmodder := item.GetLastModifiedByUser() - if lastmodder != nil { - m[fault.AddtlLastModBy] = ptr.Val(lastmodder.GetId()) - } + // lastmodder := item.GetLastModifiedByUser() + // if lastmodder != nil { + // m[fault.AddtlLastModBy] = ptr.Val(lastmodder.GetId()) + // } parent := item.GetParentReference() if parent != nil { @@ -538,10 +540,10 @@ func ItemInfo(item models.DriveItemable) map[string]any { m[fault.AddtlContainerPath] = containerPath } - malware := item.GetMalware() - if malware != nil { - m[fault.AddtlMalwareDesc] = ptr.Val(malware.GetDescription()) - } + // malware := item.GetMalware() + // if malware != nil { + // m[fault.AddtlMalwareDesc] = ptr.Val(malware.GetDescription()) + // } return m } diff --git a/src/pkg/services/m365/api/graph/errors_test.go b/src/pkg/services/m365/api/graph/errors_test.go index 2c2357071..506c87a5c 100644 --- a/src/pkg/services/m365/api/graph/errors_test.go +++ b/src/pkg/services/m365/api/graph/errors_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" graphTD "github.com/alcionai/corso/src/pkg/services/m365/api/graph/testdata" + "github.com/alcionai/corso/src/pkg/services/m365/custom" ) type GraphErrorsUnitSuite struct { @@ -532,7 +533,7 @@ func (suite *GraphErrorsUnitSuite) TestMalwareInfo() { fault.AddtlMalwareDesc: malDesc, } - assert.Equal(suite.T(), expect, ItemInfo(i)) + assert.Equal(suite.T(), expect, ItemInfo(custom.ToLiteDriveItemable(i))) } func (suite *GraphErrorsUnitSuite) TestIsErrFolderExists() { diff --git a/src/pkg/services/m365/custom/drive_item.go b/src/pkg/services/m365/custom/drive_item.go index cea91f5f3..ed6275d3a 100644 --- a/src/pkg/services/m365/custom/drive_item.go +++ b/src/pkg/services/m365/custom/drive_item.go @@ -337,3 +337,14 @@ func ToLiteDriveItemable(item models.DriveItemable) LiteDriveItemable { return di } + +func SetParentName(orig parentReferenceable, driveName string) parentReferenceable { + if orig == nil { + return nil + } + + pr := orig.(*parentRef) + pr.name = driveName + + return pr +}