larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use custom drive items during backup

Browse Source
This commit is contained in:
Abhishek Pandey 2023-12-01 22:17:24 -08:00
parent 488c3458c9
commit 1d971fb2ef
16 changed files with 78 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/internal/m365/collection/drive/collection.go
View File

@ -28,6 +28,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
const (
@ -52,7 +53,7 @@ type Collection struct {
// represents
folderPath path.Path
// M365 IDs of file items within this collection
driveItems map[string]models.DriveItemable
driveItems map[string]custom.LiteDriveItemable
// Primary M365 ID of the drive this collection was created from
driveID string
@ -172,7 +173,7 @@ func newColl(
protectedResource: resource,
folderPath: currPath,
prevPath: prevPath,
driveItems: map[string]models.DriveItemable{},
driveItems: map[string]custom.LiteDriveItemable{},
driveID: driveID,
data: dataCh,
statusUpdater: statusUpdater,
@ -191,8 +192,10 @@ func newColl(
// populated. The return values denotes if the item was previously
// present or is new one.
func (oc *Collection) Add(item models.DriveItemable) bool {
_, found := oc.driveItems[ptr.Val(item.GetId())]
oc.driveItems[ptr.Val(item.GetId())] = item
liteItem := custom.ToLiteDriveItemable(item)
_, found := oc.driveItems[ptr.Val(liteItem.GetId())]
oc.driveItems[ptr.Val(liteItem.GetId())] = liteItem
// if !found, it's a new addition
return !found
@ -277,7 +280,7 @@ func (oc Collection) DoNotMergeItems() bool {
func (oc *Collection) getDriveItemContent(
ctx context.Context,
driveID string,
item models.DriveItemable,
item custom.LiteDriveItemable,
errs *fault.Bus,
) (io.ReadCloser, error) {
var (
@ -360,7 +363,7 @@ func downloadContent(
ctx context.Context,
iaag itemAndAPIGetter,
uc getItemPropertyer,
item models.DriveItemable,
item custom.LiteDriveItemable,
driveID string,
counter *count.Bus,
) (io.ReadCloser, error) {
@ -395,7 +398,9 @@ func downloadContent(
return nil, clues.Wrap(err, "retrieving expired item")
}
content, err = downloadItem(ctx, iaag, di)
ldi := custom.ToLiteDriveItemable(di)
content, err = downloadItem(ctx, iaag, ldi)
if err != nil {
return nil, clues.Wrap(err, "content download retry")
}
@ -489,7 +494,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
wg.Add(1)
go func(item models.DriveItemable) {
go func(item custom.LiteDriveItemable) {
defer wg.Done()
defer func() { <-semaphoreCh }()
@ -513,14 +518,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
type lazyItemGetter struct {
info *details.ItemInfo
item models.DriveItemable
item custom.LiteDriveItemable
driveID string
suffix string
itemExtensionFactory []extensions.CreateItemExtensioner
contentGetter func(
ctx context.Context,
driveID string,
item models.DriveItemable,
item custom.LiteDriveItemable,
errs *fault.Bus) (io.ReadCloser, error)
}
@ -561,7 +566,7 @@ func (lig *lazyItemGetter) GetData(
func (oc *Collection) streamDriveItem(
ctx context.Context,
parentPath *path.Builder,
item models.DriveItemable,
item custom.LiteDriveItemable,
stats *driveStats,
itemExtensionFactory []extensions.CreateItemExtensioner,
errs *fault.Bus,
@ -584,7 +589,7 @@ func (oc *Collection) streamDriveItem(
"item_name", clues.Hide(itemName),
"item_size", itemSize)
item.SetParentReference(setName(item.GetParentReference(), oc.driveName))
item.SetParentReference(custom.SetParentName(item.GetParentReference(), oc.driveName))
isFile := item.GetFile() != nil

5
src/internal/m365/collection/drive/collection_test.go
View File

@ -34,6 +34,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
// ---------------------------------------------------------------------------
@ -641,7 +642,7 @@ func (suite *GetDriveItemUnitTestSuite) TestGetDriveItem_error() {
col.handler = mbh
_, err := col.getDriveItemContent(ctx, "driveID", stubItem, errs)
_, err := col.getDriveItemContent(ctx, "driveID", custom.ToLiteDriveItemable(stubItem), errs)
if test.err == nil {
assert.NoError(t, err, clues.ToCore(err))
return
@ -819,7 +820,7 @@ func (suite *GetDriveItemUnitTestSuite) TestDownloadContent() {
mbh.GetResps = resps
mbh.GetErrs = test.getErr
r, err := downloadContent(ctx, mbh, test.muc, item, driveID, count.New())
r, err := downloadContent(ctx, mbh, test.muc, custom.ToLiteDriveItemable(item), driveID, count.New())
test.expect(t, r)
test.expectErr(t, err, clues.ToCore(err))
})

5
src/internal/m365/collection/drive/collections.go
View File

@ -27,6 +27,7 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
const (
@ -955,7 +956,9 @@ func (c *Collections) processItem(
"item_is_folder", isFolder)
if item.GetMalware() != nil {
addtl := graph.ItemInfo(item)
// TODO(pandeyabs): Fix this after we move conversion logic to the top of this
// func.
addtl := graph.ItemInfo(custom.ToLiteDriveItemable(item))
skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
if isFolder {

3
src/internal/m365/collection/drive/collections_tree.go
View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
// ---------------------------------------------------------------------------
@ -464,7 +465,7 @@ func (c *Collections) addFolderToTree(
driveID,
folderID,
folderName,
graph.ItemInfo(folder))
graph.ItemInfo(custom.ToLiteDriveItemable(folder)))
logger.Ctx(ctx).Infow("malware detected")

4
src/internal/m365/collection/drive/group_handler.go
View File

@ -2,7 +2,6 @@ package drive
import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
@ -11,6 +10,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
var _ BackupHandler = &groupBackupHandler{}
@ -105,7 +105,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) {
func (h groupBackupHandler) AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item custom.LiteDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo {

7
src/internal/m365/collection/drive/handler_utils.go
View File

@ -3,12 +3,11 @@ package drive
import (
"strings"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
func getItemCreator(item models.DriveItemable) string {
func getItemCreator(item custom.LiteDriveItemable) string {
if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil {
return ""
}
@ -30,7 +29,7 @@ func getItemCreator(item models.DriveItemable) string {
return *ed.(*string)
}
func getItemDriveInfo(item models.DriveItemable) (string, string) {
func getItemDriveInfo(item custom.LiteDriveItemable) (string, string) {
if item.GetParentReference() == nil {
return "", ""
}

3
src/internal/m365/collection/drive/handlers.go
View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
type ItemInfoAugmenter interface {
@ -23,7 +24,7 @@ type ItemInfoAugmenter interface {
AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item custom.LiteDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo

16
src/internal/m365/collection/drive/item.go
View File

@ -7,7 +7,6 @@ import (
"io"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/ptr"
@ -18,6 +17,7 @@ import (
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
const (
@ -34,7 +34,7 @@ var downloadURLKeys = []string{
func downloadItem(
ctx context.Context,
ag api.Getter,
item models.DriveItemable,
item custom.LiteDriveItemable,
) (io.ReadCloser, error) {
if item == nil {
return nil, clues.New("nil item")
@ -152,7 +152,7 @@ func downloadItemMeta(
ctx context.Context,
getter GetItemPermissioner,
driveID string,
item models.DriveItemable,
item custom.LiteDriveItemable,
) (io.ReadCloser, int, error) {
meta := metadata.Metadata{
FileName: ptr.Val(item.GetName()),
@ -204,13 +204,3 @@ func driveItemWriter(
return iw, ptr.Val(icu.GetUploadUrl()), nil
}
func setName(orig models.ItemReferenceable, driveName string) models.ItemReferenceable {
if orig == nil {
return nil
}
orig.SetName(&driveName)
return orig
}

7
src/internal/m365/collection/drive/item_test.go
View File

@ -25,6 +25,7 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
type ItemIntegrationSuite struct {
@ -123,7 +124,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
}
// Read data for the file
itemData, err := downloadItem(ctx, bh, driveItem)
itemData, err := downloadItem(ctx, bh, custom.ToLiteDriveItemable(driveItem))
require.NoError(t, err, clues.ToCore(err))
size, err := io.Copy(io.Discard, itemData)
@ -462,7 +463,7 @@ func (suite *ItemUnitTestSuite) TestDownloadItem() {
mg := mockGetter{
GetFunc: test.GetFunc,
}
rc, err := downloadItem(ctx, mg, test.itemFunc())
rc, err := downloadItem(ctx, mg, custom.ToLiteDriveItemable(test.itemFunc()))
test.errorExpected(t, err, clues.ToCore(err))
test.rcExpected(t, rc)
})
@ -521,7 +522,7 @@ func (suite *ItemUnitTestSuite) TestDownloadItem_ConnectionResetErrorOnFirstRead
mg := mockGetter{
GetFunc: GetFunc,
}
rc, err := downloadItem(ctx, mg, itemFunc())
rc, err := downloadItem(ctx, mg, custom.ToLiteDriveItemable(itemFunc()))
errorExpected(t, err, clues.ToCore(err))
rcExpected(t, rc)

3
src/internal/m365/collection/drive/restore.go
View File

@ -30,6 +30,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
const (
@ -863,7 +864,7 @@ func restoreFile(
dii := ir.AugmentItemInfo(
details.ItemInfo{},
rcc.ProtectedResource,
newItem,
custom.ToLiteDriveItemable(newItem),
written,
nil)

3
src/internal/m365/collection/drive/site_handler.go
View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
type baseSiteHandler struct {
@ -33,7 +34,7 @@ func (h baseSiteHandler) NewDrivePager(
func (h baseSiteHandler) AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item custom.LiteDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo {

3
src/internal/m365/collection/drive/user_drive_handler.go
View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
// ---------------------------------------------------------------------------
@ -42,7 +43,7 @@ func (h baseUserDriveHandler) NewDrivePager(
func (h baseUserDriveHandler) AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item custom.LiteDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo {

5
src/internal/m365/service/onedrive/mock/handlers.go
View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api"
apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
// ---------------------------------------------------------------------------
@ -165,7 +166,7 @@ func (h BackupHandler[T]) NewLocationIDer(driveID string, elems ...string) detai
func (h BackupHandler[T]) AugmentItemInfo(
details.ItemInfo,
idname.Provider,
models.DriveItemable,
custom.LiteDriveItemable,
int64,
*path.Builder,
) details.ItemInfo {
@ -405,7 +406,7 @@ func (h RestoreHandler) NewDrivePager(string, []string) pagers.NonDeltaHandler[m
func (h *RestoreHandler) AugmentItemInfo(
details.ItemInfo,
idname.Provider,
models.DriveItemable,
custom.LiteDriveItemable,
int64,
*path.Builder,
) details.ItemInfo {

30
src/pkg/services/m365/api/graph/errors.go
View File

@ -10,7 +10,6 @@ import (
"syscall"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
"github.com/pkg/errors"
@ -20,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
// ---------------------------------------------------------------------------
@ -510,18 +510,20 @@ func appendIf(a []any, k string, v *string) []any {
// ItemInfo gathers potentially useful information about a drive item,
// and aggregates that data into a map.
func ItemInfo(item models.DriveItemable) map[string]any {
func ItemInfo(item custom.LiteDriveItemable) map[string]any {
m := map[string]any{}
creator := item.GetCreatedByUser()
if creator != nil {
m[fault.AddtlCreatedBy] = ptr.Val(creator.GetId())
}
// TODO(pandeyabs): These fields are not available in the LiteDriveItemable
// yet. We need to add them.
// creator := item.GetCreatedByUser()
// if creator != nil {
// m[fault.AddtlCreatedBy] = ptr.Val(creator.GetId())
// }
lastmodder := item.GetLastModifiedByUser()
if lastmodder != nil {
m[fault.AddtlLastModBy] = ptr.Val(lastmodder.GetId())
}
// lastmodder := item.GetLastModifiedByUser()
// if lastmodder != nil {
// m[fault.AddtlLastModBy] = ptr.Val(lastmodder.GetId())
// }
parent := item.GetParentReference()
if parent != nil {
@ -538,10 +540,10 @@ func ItemInfo(item models.DriveItemable) map[string]any {
m[fault.AddtlContainerPath] = containerPath
}
malware := item.GetMalware()
if malware != nil {
m[fault.AddtlMalwareDesc] = ptr.Val(malware.GetDescription())
}
// malware := item.GetMalware()
// if malware != nil {
// m[fault.AddtlMalwareDesc] = ptr.Val(malware.GetDescription())
// }
return m
}

3
src/pkg/services/m365/api/graph/errors_test.go
View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/fault"
graphTD "github.com/alcionai/corso/src/pkg/services/m365/api/graph/testdata"
"github.com/alcionai/corso/src/pkg/services/m365/custom"
)
type GraphErrorsUnitSuite struct {
@ -532,7 +533,7 @@ func (suite *GraphErrorsUnitSuite) TestMalwareInfo() {
fault.AddtlMalwareDesc: malDesc,
}
assert.Equal(suite.T(), expect, ItemInfo(i))
assert.Equal(suite.T(), expect, ItemInfo(custom.ToLiteDriveItemable(i)))
}
func (suite *GraphErrorsUnitSuite) TestIsErrFolderExists() {

11
src/pkg/services/m365/custom/drive_item.go
View File

@ -337,3 +337,14 @@ func ToLiteDriveItemable(item models.DriveItemable) LiteDriveItemable {
return di
}
func SetParentName(orig parentReferenceable, driveName string) parentReferenceable {
if orig == nil {
return nil
}
pr := orig.(*parentRef)
pr.name = driveName
return pr
}