diff --git a/src/internal/m365/collection/drive/collection.go b/src/internal/m365/collection/drive/collection.go index 8bfb70554..c8dc4969d 100644 --- a/src/internal/m365/collection/drive/collection.go +++ b/src/internal/m365/collection/drive/collection.go @@ -13,6 +13,8 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/spatialcurrent/go-lazy/pkg/lazy" + i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e "time" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" @@ -52,7 +54,7 @@ type Collection struct { // represents folderPath path.Path // M365 IDs of file items within this collection - driveItems map[string]models.DriveItemable + driveItems map[string]CorsoDriveItemable // Primary M365 ID of the drive this collection was created from driveID string @@ -92,7 +94,126 @@ type Collection struct { counter *count.Bus } -func (c *Collection) GetDriveItemsMap() map[string]models.DriveItemable { +// Replica of models.DriveItemable +type CorsoDriveItemable interface { + GetId() *string + GetName() *string + GetSize() *int64 + GetFile() models.Fileable + GetFolder() *models.Folder + GetAdditionalData() map[string]interface{} + GetParentReference() models.ItemReferenceable + SetParentReference(models.ItemReferenceable) + GetShared() models.Sharedable + GetCreatedBy() models.IdentitySetable + GetCreatedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time + GetLastModifiedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time + GetMalware() models.Malwareable + GetSharepointIds() models.SharepointIdsable + GetDeleted() models.Deletedable + GetRoot() models.Rootable +} + +type CorsoDriveItem struct { + ID *string + Name *string + Size *int64 + File models.Fileable + AdditionalData map[string]interface{} + ParentReference models.ItemReferenceable + Shared models.Sharedable + CreatedBy models.IdentitySetable + CreatedDateTime *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time + LastModifiedDateTime *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time + Malware models.Malwareable + Deleted models.Deletedable + Root models.Rootable +} + +func (c *CorsoDriveItem) GetId() *string { + return c.ID +} + +func (c *CorsoDriveItem) GetName() *string { + return c.Name +} + +func (c *CorsoDriveItem) GetSize() *int64 { + return c.Size +} + +func (c *CorsoDriveItem) GetFile() models.Fileable { + return c.File +} + +func (c *CorsoDriveItem) GetFolder() *models.Folder { + return nil +} + +func (c *CorsoDriveItem) GetAdditionalData() map[string]interface{} { + return c.AdditionalData +} + +func (c *CorsoDriveItem) GetParentReference() models.ItemReferenceable { + return c.ParentReference +} + +func (c *CorsoDriveItem) SetParentReference(parent models.ItemReferenceable) { + c.ParentReference = parent +} + +func (c *CorsoDriveItem) GetShared() models.Sharedable { + return c.Shared +} + +func (c *CorsoDriveItem) GetCreatedBy() models.IdentitySetable { + return c.CreatedBy +} + +func (c *CorsoDriveItem) GetCreatedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time { + return c.CreatedDateTime +} + +func (c *CorsoDriveItem) GetLastModifiedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time { + return c.LastModifiedDateTime +} + +func (c *CorsoDriveItem) GetMalware() models.Malwareable { + return c.Malware +} + +func (c *CorsoDriveItem) GetSharepointIds() models.SharepointIdsable { + return nil +} + +func (c *CorsoDriveItem) GetDeleted() models.Deletedable { + return c.Deleted +} + +func (c *CorsoDriveItem) GetRoot() models.Rootable { + return c.Root +} + +// models.DriveItemable to CorsoDriveItemable +func ToCorsoDriveItemable(item models.DriveItemable) CorsoDriveItemable { + return &CorsoDriveItem{ + ID: item.GetId(), + Name: item.GetName(), + Size: item.GetSize(), + File: item.GetFile(), + ParentReference: item.GetParentReference(), + Shared: item.GetShared(), + CreatedBy: item.GetCreatedBy(), + CreatedDateTime: item.GetCreatedDateTime(), + LastModifiedDateTime: item.GetLastModifiedDateTime(), + Malware: item.GetMalware(), + AdditionalData: item.GetAdditionalData(), + Deleted: item.GetDeleted(), + Root: item.GetRoot(), + } +} + +func (c *Collection) GetDriveItemsMap() map[string]CorsoDriveItemable { return c.driveItems } @@ -176,7 +297,7 @@ func newColl( protectedResource: resource, folderPath: currPath, prevPath: prevPath, - driveItems: map[string]models.DriveItemable{}, + driveItems: map[string]CorsoDriveItemable{}, driveID: driveID, data: dataCh, statusUpdater: statusUpdater, @@ -194,9 +315,13 @@ func newColl( // Adds an itemID to the collection. This will make it eligible to be // populated. The return values denotes if the item was previously // present or is new one. -func (oc *Collection) Add(item models.DriveItemable) bool { - _, found := oc.driveItems[ptr.Val(item.GetId())] - oc.driveItems[ptr.Val(item.GetId())] = item +func (oc *Collection) Add(cdi CorsoDriveItemable) bool { + // _, found := oc.driveItems[ptr.Val(item.GetId())] + // oc.driveItems[ptr.Val(item.GetId())] = item + + //cdi := ToCorsoDriveItemable(item) + _, found := oc.driveItems[ptr.Val(cdi.GetId())] + oc.driveItems[ptr.Val(cdi.GetId())] = cdi // if !found, it's a new addition return !found @@ -265,13 +390,13 @@ func (oc Collection) DoNotMergeItems() bool { func (oc *Collection) getDriveItemContent( ctx context.Context, driveID string, - item models.DriveItemable, + item CorsoDriveItemable, errs *fault.Bus, ) (io.ReadCloser, error) { - var ( - itemID = ptr.Val(item.GetId()) - itemName = ptr.Val(item.GetName()) - ) + // var ( + // itemID = ptr.Val(item.GetId()) + // itemName = ptr.Val(item.GetName()) + // ) itemData, err := downloadContent( ctx, @@ -283,7 +408,7 @@ func (oc *Collection) getDriveItemContent( if err != nil { if clues.HasLabel(err, graph.LabelsMalware) || (item != nil && item.GetMalware() != nil) { logger.CtxErr(ctx, err).With("skipped_reason", fault.SkipMalware).Info("item flagged as malware") - errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item))) + //errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item))) return nil, clues.Wrap(err, "malware item").Label(graph.LabelsSkippable) } @@ -313,12 +438,12 @@ func (oc *Collection) getDriveItemContent( CtxErr(ctx, err). With("skipped_reason", fault.SkipOneNote). Info("inaccessible one note file") - errs.AddSkip(ctx, fault.FileSkip( - fault.SkipOneNote, - driveID, - itemID, - itemName, - graph.ItemInfo(item))) + // errs.AddSkip(ctx, fault.FileSkip( + // fault.SkipOneNote, + // driveID, + // itemID, + // itemName, + // graph.ItemInfo(item))) return nil, clues.Wrap(err, "inaccesible oneNote item").Label(graph.LabelsSkippable) } @@ -348,7 +473,7 @@ func downloadContent( ctx context.Context, iaag itemAndAPIGetter, uc getItemPropertyer, - item models.DriveItemable, + item CorsoDriveItemable, driveID string, counter *count.Bus, ) (io.ReadCloser, error) { @@ -383,7 +508,9 @@ func downloadContent( return nil, clues.Wrap(err, "retrieving expired item") } - content, err = downloadItem(ctx, iaag, di) + cdi := ToCorsoDriveItemable(di) + + content, err = downloadItem(ctx, iaag, cdi) if err != nil { return nil, clues.Wrap(err, "content download retry") } @@ -477,7 +604,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) { wg.Add(1) - go func(item models.DriveItemable) { + go func(item CorsoDriveItemable) { defer wg.Done() defer func() { <-semaphoreCh }() @@ -501,14 +628,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) { type lazyItemGetter struct { info *details.ItemInfo - item models.DriveItemable + item CorsoDriveItemable driveID string suffix string itemExtensionFactory []extensions.CreateItemExtensioner contentGetter func( ctx context.Context, driveID string, - item models.DriveItemable, + item CorsoDriveItemable, errs *fault.Bus) (io.ReadCloser, error) } @@ -549,7 +676,7 @@ func (lig *lazyItemGetter) GetData( func (oc *Collection) streamDriveItem( ctx context.Context, parentPath *path.Builder, - item models.DriveItemable, + item CorsoDriveItemable, stats *driveStats, itemExtensionFactory []extensions.CreateItemExtensioner, errs *fault.Bus, diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index ce93a9fa8..d580b98f1 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -21,7 +21,6 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -696,7 +695,7 @@ func (c *Collections) handleDelete( func (c *Collections) getCollectionPath( driveID string, - item models.DriveItemable, + item CorsoDriveItemable, ) (path.Path, error) { var ( pb = odConsts.DriveFolderPrefixBuilder(driveID) @@ -845,7 +844,7 @@ func (c *Collections) PopulateDriveCollections( func (c *Collections) processItem( ctx context.Context, - item models.DriveItemable, + di models.DriveItemable, driveID, driveName string, oldPrevPaths, currPrevPaths, newPrevPaths map[string]string, seenFolders map[string]string, @@ -855,10 +854,12 @@ func (c *Collections) processItem( counter *count.Bus, skipper fault.AddSkipper, ) error { + item := ToCorsoDriveItemable(di) + var ( itemID = ptr.Val(item.GetId()) itemName = ptr.Val(item.GetName()) - isFolder = item.GetFolder() != nil || item.GetPackageEscaped() != nil + isFolder = item.GetFolder() != nil ) ctx = clues.Add( @@ -868,16 +869,16 @@ func (c *Collections) processItem( "item_is_folder", isFolder) if item.GetMalware() != nil { - addtl := graph.ItemInfo(item) - skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) + // addtl := graph.ItemInfo(item) + // skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) - if isFolder { - skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) - } + // if isFolder { + // skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) + // } - skipper.AddSkip(ctx, skip) - logger.Ctx(ctx).Infow("malware detected", "item_details", addtl) - counter.Inc(count.Malware) + // skipper.AddSkip(ctx, skip) + // logger.Ctx(ctx).Infow("malware detected", "item_details", addtl) + // counter.Inc(count.Malware) return nil } @@ -948,19 +949,9 @@ func (c *Collections) processItem( return nil } - isPackage := item.GetPackageEscaped() != nil - if isPackage { - counter.Inc(count.Packages) - // mark this path as a package type for all other collections. - // any subfolder should get marked as a childOfPackage below. - topLevelPackages[collectionPath.String()] = struct{}{} - } else { - counter.Inc(count.Folders) - } - - childOfPackage := filters. - PathPrefix(maps.Keys(topLevelPackages)). - Compare(collectionPath.String()) + // childOfPackage := filters. + // PathPrefix(maps.Keys(topLevelPackages)). + // Compare(collectionPath.String()) // This check is to ensure that if a folder was deleted and // recreated multiple times between a backup, we only use the @@ -996,7 +987,7 @@ func (c *Collections) processItem( driveID, c.statusUpdater, c.ctrl, - isPackage || childOfPackage, + false, invalidPrevDelta || collPathAlreadyExists, nil, counter.Local()) diff --git a/src/internal/m365/collection/drive/group_handler.go b/src/internal/m365/collection/drive/group_handler.go index a603ce748..e723e0709 100644 --- a/src/internal/m365/collection/drive/group_handler.go +++ b/src/internal/m365/collection/drive/group_handler.go @@ -2,7 +2,6 @@ package drive import ( "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" @@ -105,7 +104,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) { func (h groupBackupHandler) AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item CorsoDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo { diff --git a/src/internal/m365/collection/drive/handler_utils.go b/src/internal/m365/collection/drive/handler_utils.go index dd1e415b4..96eba0cbe 100644 --- a/src/internal/m365/collection/drive/handler_utils.go +++ b/src/internal/m365/collection/drive/handler_utils.go @@ -3,12 +3,10 @@ package drive import ( "strings" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/alcionai/corso/src/internal/common/ptr" ) -func getItemCreator(item models.DriveItemable) string { +func getItemCreator(item CorsoDriveItemable) string { if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil { return "" } @@ -30,7 +28,7 @@ func getItemCreator(item models.DriveItemable) string { return *ed.(*string) } -func getItemDriveInfo(item models.DriveItemable) (string, string) { +func getItemDriveInfo(item CorsoDriveItemable) (string, string) { if item.GetParentReference() == nil { return "", "" } diff --git a/src/internal/m365/collection/drive/handlers.go b/src/internal/m365/collection/drive/handlers.go index 76c147144..4ec64ad63 100644 --- a/src/internal/m365/collection/drive/handlers.go +++ b/src/internal/m365/collection/drive/handlers.go @@ -23,7 +23,7 @@ type ItemInfoAugmenter interface { AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item CorsoDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo diff --git a/src/internal/m365/collection/drive/item.go b/src/internal/m365/collection/drive/item.go index 9fad7cd8e..150d00dc6 100644 --- a/src/internal/m365/collection/drive/item.go +++ b/src/internal/m365/collection/drive/item.go @@ -34,7 +34,7 @@ var downloadURLKeys = []string{ func downloadItem( ctx context.Context, ag api.Getter, - item models.DriveItemable, + item CorsoDriveItemable, ) (io.ReadCloser, error) { if item == nil { return nil, clues.New("nil item") @@ -152,7 +152,7 @@ func downloadItemMeta( ctx context.Context, getter GetItemPermissioner, driveID string, - item models.DriveItemable, + item CorsoDriveItemable, ) (io.ReadCloser, int, error) { meta := metadata.Metadata{ FileName: ptr.Val(item.GetName()), diff --git a/src/internal/m365/collection/drive/restore.go b/src/internal/m365/collection/drive/restore.go index c842baa18..4510ca81e 100644 --- a/src/internal/m365/collection/drive/restore.go +++ b/src/internal/m365/collection/drive/restore.go @@ -863,7 +863,7 @@ func restoreFile( dii := ir.AugmentItemInfo( details.ItemInfo{}, rcc.ProtectedResource, - newItem, + nil, written, nil) diff --git a/src/internal/m365/collection/drive/site_handler.go b/src/internal/m365/collection/drive/site_handler.go index 2500b565a..5bba7d977 100644 --- a/src/internal/m365/collection/drive/site_handler.go +++ b/src/internal/m365/collection/drive/site_handler.go @@ -33,7 +33,7 @@ func (h baseSiteHandler) NewDrivePager( func (h baseSiteHandler) AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item CorsoDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo { diff --git a/src/internal/m365/collection/drive/user_drive_handler.go b/src/internal/m365/collection/drive/user_drive_handler.go index dad4a1f6c..e38084ff9 100644 --- a/src/internal/m365/collection/drive/user_drive_handler.go +++ b/src/internal/m365/collection/drive/user_drive_handler.go @@ -42,7 +42,7 @@ func (h baseUserDriveHandler) NewDrivePager( func (h baseUserDriveHandler) AugmentItemInfo( dii details.ItemInfo, resource idname.Provider, - item models.DriveItemable, + item CorsoDriveItemable, size int64, parentPath *path.Builder, ) details.ItemInfo {