larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add custom drive itemable

Browse Source
This commit is contained in:
Abhishek Pandey 2023-11-18 00:37:39 -08:00
parent b1cf23bee3
commit f1f1be5b9c
9 changed files with 177 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
src/internal/m365/collection/drive/collection.go
View File

@ -13,6 +13,8 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/spatialcurrent/go-lazy/pkg/lazy"
i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e "time"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/data"
@ -52,7 +54,7 @@ type Collection struct {
// represents
folderPath path.Path
// M365 IDs of file items within this collection
driveItems map[string]models.DriveItemable
driveItems map[string]CorsoDriveItemable
// Primary M365 ID of the drive this collection was created from
driveID string
@ -92,7 +94,126 @@ type Collection struct {
counter *count.Bus
}
func (c *Collection) GetDriveItemsMap() map[string]models.DriveItemable {
// Replica of models.DriveItemable
type CorsoDriveItemable interface {
GetId() *string
GetName() *string
GetSize() *int64
GetFile() models.Fileable
GetFolder() *models.Folder
GetAdditionalData() map[string]interface{}
GetParentReference() models.ItemReferenceable
SetParentReference(models.ItemReferenceable)
GetShared() models.Sharedable
GetCreatedBy() models.IdentitySetable
GetCreatedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
GetLastModifiedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
GetMalware() models.Malwareable
GetSharepointIds() models.SharepointIdsable
GetDeleted() models.Deletedable
GetRoot() models.Rootable
}
type CorsoDriveItem struct {
ID *string
Name *string
Size *int64
File models.Fileable
AdditionalData map[string]interface{}
ParentReference models.ItemReferenceable
Shared models.Sharedable
CreatedBy models.IdentitySetable
CreatedDateTime *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
LastModifiedDateTime *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
Malware models.Malwareable
Deleted models.Deletedable
Root models.Rootable
}
func (c *CorsoDriveItem) GetId() *string {
return c.ID
}
func (c *CorsoDriveItem) GetName() *string {
return c.Name
}
func (c *CorsoDriveItem) GetSize() *int64 {
return c.Size
}
func (c *CorsoDriveItem) GetFile() models.Fileable {
return c.File
}
func (c *CorsoDriveItem) GetFolder() *models.Folder {
return nil
}
func (c *CorsoDriveItem) GetAdditionalData() map[string]interface{} {
return c.AdditionalData
}
func (c *CorsoDriveItem) GetParentReference() models.ItemReferenceable {
return c.ParentReference
}
func (c *CorsoDriveItem) SetParentReference(parent models.ItemReferenceable) {
c.ParentReference = parent
}
func (c *CorsoDriveItem) GetShared() models.Sharedable {
return c.Shared
}
func (c *CorsoDriveItem) GetCreatedBy() models.IdentitySetable {
return c.CreatedBy
}
func (c *CorsoDriveItem) GetCreatedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time {
return c.CreatedDateTime
}
func (c *CorsoDriveItem) GetLastModifiedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time {
return c.LastModifiedDateTime
}
func (c *CorsoDriveItem) GetMalware() models.Malwareable {
return c.Malware
}
func (c *CorsoDriveItem) GetSharepointIds() models.SharepointIdsable {
return nil
}
func (c *CorsoDriveItem) GetDeleted() models.Deletedable {
return c.Deleted
}
func (c *CorsoDriveItem) GetRoot() models.Rootable {
return c.Root
}
// models.DriveItemable to CorsoDriveItemable
func ToCorsoDriveItemable(item models.DriveItemable) CorsoDriveItemable {
return &CorsoDriveItem{
ID: item.GetId(),
Name: item.GetName(),
Size: item.GetSize(),
File: item.GetFile(),
ParentReference: item.GetParentReference(),
Shared: item.GetShared(),
CreatedBy: item.GetCreatedBy(),
CreatedDateTime: item.GetCreatedDateTime(),
LastModifiedDateTime: item.GetLastModifiedDateTime(),
Malware: item.GetMalware(),
AdditionalData: item.GetAdditionalData(),
Deleted: item.GetDeleted(),
Root: item.GetRoot(),
}
}
func (c *Collection) GetDriveItemsMap() map[string]CorsoDriveItemable {
return c.driveItems
}
@ -176,7 +297,7 @@ func newColl(
protectedResource: resource,
folderPath: currPath,
prevPath: prevPath,
driveItems: map[string]models.DriveItemable{},
driveItems: map[string]CorsoDriveItemable{},
driveID: driveID,
data: dataCh,
statusUpdater: statusUpdater,
@ -194,9 +315,13 @@ func newColl(
// Adds an itemID to the collection. This will make it eligible to be
// populated. The return values denotes if the item was previously
// present or is new one.
func (oc *Collection) Add(item models.DriveItemable) bool {
_, found := oc.driveItems[ptr.Val(item.GetId())]
oc.driveItems[ptr.Val(item.GetId())] = item
func (oc *Collection) Add(cdi CorsoDriveItemable) bool {
// _, found := oc.driveItems[ptr.Val(item.GetId())]
// oc.driveItems[ptr.Val(item.GetId())] = item
//cdi := ToCorsoDriveItemable(item)
_, found := oc.driveItems[ptr.Val(cdi.GetId())]
oc.driveItems[ptr.Val(cdi.GetId())] = cdi
// if !found, it's a new addition
return !found
@ -265,13 +390,13 @@ func (oc Collection) DoNotMergeItems() bool {
func (oc *Collection) getDriveItemContent(
ctx context.Context,
driveID string,
item models.DriveItemable,
item CorsoDriveItemable,
errs *fault.Bus,
) (io.ReadCloser, error) {
var (
itemID = ptr.Val(item.GetId())
itemName = ptr.Val(item.GetName())
)
// var (
// itemID = ptr.Val(item.GetId())
// itemName = ptr.Val(item.GetName())
// )
itemData, err := downloadContent(
ctx,
@ -283,7 +408,7 @@ func (oc *Collection) getDriveItemContent(
if err != nil {
if clues.HasLabel(err, graph.LabelsMalware) || (item != nil && item.GetMalware() != nil) {
logger.CtxErr(ctx, err).With("skipped_reason", fault.SkipMalware).Info("item flagged as malware")
errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
//errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
return nil, clues.Wrap(err, "malware item").Label(graph.LabelsSkippable)
}
@ -313,12 +438,12 @@ func (oc *Collection) getDriveItemContent(
CtxErr(ctx, err).
With("skipped_reason", fault.SkipOneNote).
Info("inaccessible one note file")
errs.AddSkip(ctx, fault.FileSkip(
fault.SkipOneNote,
driveID,
itemID,
itemName,
graph.ItemInfo(item)))
// errs.AddSkip(ctx, fault.FileSkip(
// fault.SkipOneNote,
// driveID,
// itemID,
// itemName,
// graph.ItemInfo(item)))
return nil, clues.Wrap(err, "inaccesible oneNote item").Label(graph.LabelsSkippable)
}
@ -348,7 +473,7 @@ func downloadContent(
ctx context.Context,
iaag itemAndAPIGetter,
uc getItemPropertyer,
item models.DriveItemable,
item CorsoDriveItemable,
driveID string,
counter *count.Bus,
) (io.ReadCloser, error) {
@ -383,7 +508,9 @@ func downloadContent(
return nil, clues.Wrap(err, "retrieving expired item")
}
content, err = downloadItem(ctx, iaag, di)
cdi := ToCorsoDriveItemable(di)
content, err = downloadItem(ctx, iaag, cdi)
if err != nil {
return nil, clues.Wrap(err, "content download retry")
}
@ -477,7 +604,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
wg.Add(1)
go func(item models.DriveItemable) {
go func(item CorsoDriveItemable) {
defer wg.Done()
defer func() { <-semaphoreCh }()
@ -501,14 +628,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
type lazyItemGetter struct {
info *details.ItemInfo
item models.DriveItemable
item CorsoDriveItemable
driveID string
suffix string
itemExtensionFactory []extensions.CreateItemExtensioner
contentGetter func(
ctx context.Context,
driveID string,
item models.DriveItemable,
item CorsoDriveItemable,
errs *fault.Bus) (io.ReadCloser, error)
}
@ -549,7 +676,7 @@ func (lig *lazyItemGetter) GetData(
func (oc *Collection) streamDriveItem(
ctx context.Context,
parentPath *path.Builder,
item models.DriveItemable,
item CorsoDriveItemable,
stats *driveStats,
itemExtensionFactory []extensions.CreateItemExtensioner,
errs *fault.Bus,

43
src/internal/m365/collection/drive/collections.go
View File

@ -21,7 +21,6 @@ import (
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
@ -696,7 +695,7 @@ func (c *Collections) handleDelete(
func (c *Collections) getCollectionPath(
driveID string,
item models.DriveItemable,
item CorsoDriveItemable,
) (path.Path, error) {
var (
pb = odConsts.DriveFolderPrefixBuilder(driveID)
@ -845,7 +844,7 @@ func (c *Collections) PopulateDriveCollections(
func (c *Collections) processItem(
ctx context.Context,
item models.DriveItemable,
di models.DriveItemable,
driveID, driveName string,
oldPrevPaths, currPrevPaths, newPrevPaths map[string]string,
seenFolders map[string]string,
@ -855,10 +854,12 @@ func (c *Collections) processItem(
counter *count.Bus,
skipper fault.AddSkipper,
) error {
item := ToCorsoDriveItemable(di)
var (
itemID = ptr.Val(item.GetId())
itemName = ptr.Val(item.GetName())
isFolder = item.GetFolder() != nil || item.GetPackageEscaped() != nil
isFolder = item.GetFolder() != nil
)
ctx = clues.Add(
@ -868,16 +869,16 @@ func (c *Collections) processItem(
"item_is_folder", isFolder)
if item.GetMalware() != nil {
addtl := graph.ItemInfo(item)
skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
// addtl := graph.ItemInfo(item)
// skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
if isFolder {
skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
}
// if isFolder {
// skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
// }
skipper.AddSkip(ctx, skip)
logger.Ctx(ctx).Infow("malware detected", "item_details", addtl)
counter.Inc(count.Malware)
// skipper.AddSkip(ctx, skip)
// logger.Ctx(ctx).Infow("malware detected", "item_details", addtl)
// counter.Inc(count.Malware)
return nil
}
@ -948,19 +949,9 @@ func (c *Collections) processItem(
return nil
}
isPackage := item.GetPackageEscaped() != nil
if isPackage {
counter.Inc(count.Packages)
// mark this path as a package type for all other collections.
// any subfolder should get marked as a childOfPackage below.
topLevelPackages[collectionPath.String()] = struct{}{}
} else {
counter.Inc(count.Folders)
}
childOfPackage := filters.
PathPrefix(maps.Keys(topLevelPackages)).
Compare(collectionPath.String())
// childOfPackage := filters.
// PathPrefix(maps.Keys(topLevelPackages)).
// Compare(collectionPath.String())
// This check is to ensure that if a folder was deleted and
// recreated multiple times between a backup, we only use the
@ -996,7 +987,7 @@ func (c *Collections) processItem(
driveID,
c.statusUpdater,
c.ctrl,
isPackage || childOfPackage,
false,
invalidPrevDelta || collPathAlreadyExists,
nil,
counter.Local())

3
src/internal/m365/collection/drive/group_handler.go
View File

@ -2,7 +2,6 @@ package drive
import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
@ -105,7 +104,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) {
func (h groupBackupHandler) AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item CorsoDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo {

6
src/internal/m365/collection/drive/handler_utils.go
View File

@ -3,12 +3,10 @@ package drive
import (
"strings"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/ptr"
)
func getItemCreator(item models.DriveItemable) string {
func getItemCreator(item CorsoDriveItemable) string {
if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil {
return ""
}
@ -30,7 +28,7 @@ func getItemCreator(item models.DriveItemable) string {
return *ed.(*string)
}
func getItemDriveInfo(item models.DriveItemable) (string, string) {
func getItemDriveInfo(item CorsoDriveItemable) (string, string) {
if item.GetParentReference() == nil {
return "", ""
}

2
src/internal/m365/collection/drive/handlers.go
View File

@ -23,7 +23,7 @@ type ItemInfoAugmenter interface {
AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item CorsoDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo

4
src/internal/m365/collection/drive/item.go
View File

@ -34,7 +34,7 @@ var downloadURLKeys = []string{
func downloadItem(
ctx context.Context,
ag api.Getter,
item models.DriveItemable,
item CorsoDriveItemable,
) (io.ReadCloser, error) {
if item == nil {
return nil, clues.New("nil item")
@ -152,7 +152,7 @@ func downloadItemMeta(
ctx context.Context,
getter GetItemPermissioner,
driveID string,
item models.DriveItemable,
item CorsoDriveItemable,
) (io.ReadCloser, int, error) {
meta := metadata.Metadata{
FileName: ptr.Val(item.GetName()),

2
src/internal/m365/collection/drive/restore.go
View File

@ -863,7 +863,7 @@ func restoreFile(
dii := ir.AugmentItemInfo(
details.ItemInfo{},
rcc.ProtectedResource,
newItem,
nil,
written,
nil)

2
src/internal/m365/collection/drive/site_handler.go
View File

@ -33,7 +33,7 @@ func (h baseSiteHandler) NewDrivePager(
func (h baseSiteHandler) AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item CorsoDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo {

2
src/internal/m365/collection/drive/user_drive_handler.go
View File

@ -42,7 +42,7 @@ func (h baseUserDriveHandler) NewDrivePager(
func (h baseUserDriveHandler) AugmentItemInfo(
dii details.ItemInfo,
resource idname.Provider,
item models.DriveItemable,
item CorsoDriveItemable,
size int64,
parentPath *path.Builder,
) details.ItemInfo {