Add custom drive itemable

src/internal/m365/collection/drive/collection.go

@@ -13,6 +13,8 @@ import (
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/spatialcurrent/go-lazy/pkg/lazy"
 
+	i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e "time"
+
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/internal/data"
@@ -52,7 +54,7 @@ type Collection struct {
 	// represents
 	folderPath path.Path
 	// M365 IDs of file items within this collection
-	driveItems map[string]models.DriveItemable
+	driveItems map[string]CorsoDriveItemable
 
 	// Primary M365 ID of the drive this collection was created from
 	driveID   string
@@ -92,7 +94,126 @@ type Collection struct {
 	counter *count.Bus
 }
 
-func (c *Collection) GetDriveItemsMap() map[string]models.DriveItemable {
+// Replica of models.DriveItemable
+type CorsoDriveItemable interface {
+	GetId() *string
+	GetName() *string
+	GetSize() *int64
+	GetFile() models.Fileable
+	GetFolder() *models.Folder
+	GetAdditionalData() map[string]interface{}
+	GetParentReference() models.ItemReferenceable
+	SetParentReference(models.ItemReferenceable)
+	GetShared() models.Sharedable
+	GetCreatedBy() models.IdentitySetable
+	GetCreatedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
+	GetLastModifiedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
+	GetMalware() models.Malwareable
+	GetSharepointIds() models.SharepointIdsable
+	GetDeleted() models.Deletedable
+	GetRoot() models.Rootable
+}
+
+type CorsoDriveItem struct {
+	ID                   *string
+	Name                 *string
+	Size                 *int64
+	File                 models.Fileable
+	AdditionalData       map[string]interface{}
+	ParentReference      models.ItemReferenceable
+	Shared               models.Sharedable
+	CreatedBy            models.IdentitySetable
+	CreatedDateTime      *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
+	LastModifiedDateTime *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time
+	Malware              models.Malwareable
+	Deleted              models.Deletedable
+	Root                 models.Rootable
+}
+
+func (c *CorsoDriveItem) GetId() *string {
+	return c.ID
+}
+
+func (c *CorsoDriveItem) GetName() *string {
+	return c.Name
+}
+
+func (c *CorsoDriveItem) GetSize() *int64 {
+	return c.Size
+}
+
+func (c *CorsoDriveItem) GetFile() models.Fileable {
+	return c.File
+}
+
+func (c *CorsoDriveItem) GetFolder() *models.Folder {
+	return nil
+}
+
+func (c *CorsoDriveItem) GetAdditionalData() map[string]interface{} {
+	return c.AdditionalData
+}
+
+func (c *CorsoDriveItem) GetParentReference() models.ItemReferenceable {
+	return c.ParentReference
+}
+
+func (c *CorsoDriveItem) SetParentReference(parent models.ItemReferenceable) {
+	c.ParentReference = parent
+}
+
+func (c *CorsoDriveItem) GetShared() models.Sharedable {
+	return c.Shared
+}
+
+func (c *CorsoDriveItem) GetCreatedBy() models.IdentitySetable {
+	return c.CreatedBy
+}
+
+func (c *CorsoDriveItem) GetCreatedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time {
+	return c.CreatedDateTime
+}
+
+func (c *CorsoDriveItem) GetLastModifiedDateTime() *i336074805fc853987abe6f7fe3ad97a6a6f3077a16391fec744f671a015fbd7e.Time {
+	return c.LastModifiedDateTime
+}
+
+func (c *CorsoDriveItem) GetMalware() models.Malwareable {
+	return c.Malware
+}
+
+func (c *CorsoDriveItem) GetSharepointIds() models.SharepointIdsable {
+	return nil
+}
+
+func (c *CorsoDriveItem) GetDeleted() models.Deletedable {
+	return c.Deleted
+}
+
+func (c *CorsoDriveItem) GetRoot() models.Rootable {
+	return c.Root
+}
+
+// models.DriveItemable to CorsoDriveItemable
+func ToCorsoDriveItemable(item models.DriveItemable) CorsoDriveItemable {
+	return &CorsoDriveItem{
+		ID:                   item.GetId(),
+		Name:                 item.GetName(),
+		Size:                 item.GetSize(),
+		File:                 item.GetFile(),
+		ParentReference:      item.GetParentReference(),
+		Shared:               item.GetShared(),
+		CreatedBy:            item.GetCreatedBy(),
+		CreatedDateTime:      item.GetCreatedDateTime(),
+		LastModifiedDateTime: item.GetLastModifiedDateTime(),
+		Malware:              item.GetMalware(),
+		AdditionalData:       item.GetAdditionalData(),
+		Deleted:              item.GetDeleted(),
+		Root:                 item.GetRoot(),
+	}
+}
+
+func (c *Collection) GetDriveItemsMap() map[string]CorsoDriveItemable {
 	return c.driveItems
 }
 
@@ -176,7 +297,7 @@ func newColl(
 		protectedResource:         resource,
 		folderPath:                currPath,
 		prevPath:                  prevPath,
-		driveItems:                map[string]models.DriveItemable{},
+		driveItems:                map[string]CorsoDriveItemable{},
 		driveID:                   driveID,
 		data:                      dataCh,
 		statusUpdater:             statusUpdater,
@@ -194,9 +315,13 @@ func newColl(
 // Adds an itemID to the collection.  This will make it eligible to be
 // populated. The return values denotes if the item was previously
 // present or is new one.
-func (oc *Collection) Add(item models.DriveItemable) bool {
+func (oc *Collection) Add(cdi CorsoDriveItemable) bool {
-	_, found := oc.driveItems[ptr.Val(item.GetId())]
+	// _, found := oc.driveItems[ptr.Val(item.GetId())]
-	oc.driveItems[ptr.Val(item.GetId())] = item
+	// oc.driveItems[ptr.Val(item.GetId())] = item
+
+	//cdi := ToCorsoDriveItemable(item)
+	_, found := oc.driveItems[ptr.Val(cdi.GetId())]
+	oc.driveItems[ptr.Val(cdi.GetId())] = cdi
 
 	// if !found, it's a new addition
 	return !found
@@ -265,13 +390,13 @@ func (oc Collection) DoNotMergeItems() bool {
 func (oc *Collection) getDriveItemContent(
 	ctx context.Context,
 	driveID string,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	errs *fault.Bus,
 ) (io.ReadCloser, error) {
-	var (
+	// var (
-		itemID   = ptr.Val(item.GetId())
+	// 	itemID   = ptr.Val(item.GetId())
-		itemName = ptr.Val(item.GetName())
+	// 	itemName = ptr.Val(item.GetName())
-	)
+	// )
 
 	itemData, err := downloadContent(
 		ctx,
@@ -283,7 +408,7 @@ func (oc *Collection) getDriveItemContent(
 	if err != nil {
 		if clues.HasLabel(err, graph.LabelsMalware) || (item != nil && item.GetMalware() != nil) {
 			logger.CtxErr(ctx, err).With("skipped_reason", fault.SkipMalware).Info("item flagged as malware")
-			errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
+			//errs.AddSkip(ctx, fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, graph.ItemInfo(item)))
 
 			return nil, clues.Wrap(err, "malware item").Label(graph.LabelsSkippable)
 		}
@@ -313,12 +438,12 @@ func (oc *Collection) getDriveItemContent(
 				CtxErr(ctx, err).
 				With("skipped_reason", fault.SkipOneNote).
 				Info("inaccessible one note file")
-			errs.AddSkip(ctx, fault.FileSkip(
+			// errs.AddSkip(ctx, fault.FileSkip(
-				fault.SkipOneNote,
+			// 	fault.SkipOneNote,
-				driveID,
+			// 	driveID,
-				itemID,
+			// 	itemID,
-				itemName,
+			// 	itemName,
-				graph.ItemInfo(item)))
+			// 	graph.ItemInfo(item)))
 
 			return nil, clues.Wrap(err, "inaccesible oneNote item").Label(graph.LabelsSkippable)
 		}
@@ -348,7 +473,7 @@ func downloadContent(
 	ctx context.Context,
 	iaag itemAndAPIGetter,
 	uc getItemPropertyer,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	driveID string,
 	counter *count.Bus,
 ) (io.ReadCloser, error) {
@@ -383,7 +508,9 @@ func downloadContent(
 		return nil, clues.Wrap(err, "retrieving expired item")
 	}
 
-	content, err = downloadItem(ctx, iaag, di)
+	cdi := ToCorsoDriveItemable(di)
+
+	content, err = downloadItem(ctx, iaag, cdi)
 	if err != nil {
 		return nil, clues.Wrap(err, "content download retry")
 	}
@@ -477,7 +604,7 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 
 		wg.Add(1)
 
-		go func(item models.DriveItemable) {
+		go func(item CorsoDriveItemable) {
 			defer wg.Done()
 			defer func() { <-semaphoreCh }()
 
@@ -501,14 +628,14 @@ func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 
 type lazyItemGetter struct {
 	info                 *details.ItemInfo
-	item                 models.DriveItemable
+	item                 CorsoDriveItemable
 	driveID              string
 	suffix               string
 	itemExtensionFactory []extensions.CreateItemExtensioner
 	contentGetter        func(
 		ctx context.Context,
 		driveID string,
-		item models.DriveItemable,
+		item CorsoDriveItemable,
 		errs *fault.Bus) (io.ReadCloser, error)
 }
 
@@ -549,7 +676,7 @@ func (lig *lazyItemGetter) GetData(
 func (oc *Collection) streamDriveItem(
 	ctx context.Context,
 	parentPath *path.Builder,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	stats *driveStats,
 	itemExtensionFactory []extensions.CreateItemExtensioner,
 	errs *fault.Bus,

src/internal/m365/collection/drive/collections.go

@@ -21,7 +21,6 @@ import (
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/count"
 	"github.com/alcionai/corso/src/pkg/fault"
-	"github.com/alcionai/corso/src/pkg/filters"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/services/m365/api"
@@ -696,7 +695,7 @@ func (c *Collections) handleDelete(
 
 func (c *Collections) getCollectionPath(
 	driveID string,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 ) (path.Path, error) {
 	var (
 		pb     = odConsts.DriveFolderPrefixBuilder(driveID)
@@ -845,7 +844,7 @@ func (c *Collections) PopulateDriveCollections(
 
 func (c *Collections) processItem(
 	ctx context.Context,
-	item models.DriveItemable,
+	di models.DriveItemable,
 	driveID, driveName string,
 	oldPrevPaths, currPrevPaths, newPrevPaths map[string]string,
 	seenFolders map[string]string,
@@ -855,10 +854,12 @@ func (c *Collections) processItem(
 	counter *count.Bus,
 	skipper fault.AddSkipper,
 ) error {
+	item := ToCorsoDriveItemable(di)
+
 	var (
 		itemID   = ptr.Val(item.GetId())
 		itemName = ptr.Val(item.GetName())
-		isFolder = item.GetFolder() != nil || item.GetPackageEscaped() != nil
+		isFolder = item.GetFolder() != nil
 	)
 
 	ctx = clues.Add(
@@ -868,16 +869,16 @@ func (c *Collections) processItem(
 		"item_is_folder", isFolder)
 
 	if item.GetMalware() != nil {
-		addtl := graph.ItemInfo(item)
+		// addtl := graph.ItemInfo(item)
-		skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
+		// skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
 
-		if isFolder {
+		// if isFolder {
-			skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
+		// 	skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
-		}
+		// }
 
-		skipper.AddSkip(ctx, skip)
+		// skipper.AddSkip(ctx, skip)
-		logger.Ctx(ctx).Infow("malware detected", "item_details", addtl)
+		// logger.Ctx(ctx).Infow("malware detected", "item_details", addtl)
-		counter.Inc(count.Malware)
+		// counter.Inc(count.Malware)
 
 		return nil
 	}
@@ -948,19 +949,9 @@ func (c *Collections) processItem(
 			return nil
 		}
 
-		isPackage := item.GetPackageEscaped() != nil
+		// childOfPackage := filters.
-		if isPackage {
+		// 	PathPrefix(maps.Keys(topLevelPackages)).
-			counter.Inc(count.Packages)
+		// 	Compare(collectionPath.String())
-			// mark this path as a package type for all other collections.
-			// any subfolder should get marked as a childOfPackage below.
-			topLevelPackages[collectionPath.String()] = struct{}{}
-		} else {
-			counter.Inc(count.Folders)
-		}
-
-		childOfPackage := filters.
-			PathPrefix(maps.Keys(topLevelPackages)).
-			Compare(collectionPath.String())
 
 		// This check is to ensure that if a folder was deleted and
 		// recreated multiple times between a backup, we only use the
@@ -996,7 +987,7 @@ func (c *Collections) processItem(
 			driveID,
 			c.statusUpdater,
 			c.ctrl,
-			isPackage || childOfPackage,
+			false,
 			invalidPrevDelta || collPathAlreadyExists,
 			nil,
 			counter.Local())

src/internal/m365/collection/drive/group_handler.go

@@ -2,7 +2,6 @@ package drive
 
 import (
 	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/common/ptr"
@@ -105,7 +104,7 @@ func (h groupBackupHandler) SitePathPrefix(tenantID string) (path.Path, error) {
 func (h groupBackupHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/m365/collection/drive/handler_utils.go

@@ -3,12 +3,10 @@ package drive
 import (
 	"strings"
 
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
-
 	"github.com/alcionai/corso/src/internal/common/ptr"
 )
 
-func getItemCreator(item models.DriveItemable) string {
+func getItemCreator(item CorsoDriveItemable) string {
 	if item.GetCreatedBy() == nil || item.GetCreatedBy().GetUser() == nil {
 		return ""
 	}
@@ -30,7 +28,7 @@ func getItemCreator(item models.DriveItemable) string {
 	return *ed.(*string)
 }
 
-func getItemDriveInfo(item models.DriveItemable) (string, string) {
+func getItemDriveInfo(item CorsoDriveItemable) (string, string) {
 	if item.GetParentReference() == nil {
 		return "", ""
 	}

src/internal/m365/collection/drive/handlers.go

@@ -23,7 +23,7 @@ type ItemInfoAugmenter interface {
 	AugmentItemInfo(
 		dii details.ItemInfo,
 		resource idname.Provider,
-		item models.DriveItemable,
+		item CorsoDriveItemable,
 		size int64,
 		parentPath *path.Builder,
 	) details.ItemInfo

src/internal/m365/collection/drive/item.go

@@ -34,7 +34,7 @@ var downloadURLKeys = []string{
 func downloadItem(
 	ctx context.Context,
 	ag api.Getter,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 ) (io.ReadCloser, error) {
 	if item == nil {
 		return nil, clues.New("nil item")
@@ -152,7 +152,7 @@ func downloadItemMeta(
 	ctx context.Context,
 	getter GetItemPermissioner,
 	driveID string,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 ) (io.ReadCloser, int, error) {
 	meta := metadata.Metadata{
 		FileName:    ptr.Val(item.GetName()),

src/internal/m365/collection/drive/restore.go

@@ -863,7 +863,7 @@ func restoreFile(
 	dii := ir.AugmentItemInfo(
 		details.ItemInfo{},
 		rcc.ProtectedResource,
-		newItem,
+		nil,
 		written,
 		nil)
 

src/internal/m365/collection/drive/site_handler.go

@@ -33,7 +33,7 @@ func (h baseSiteHandler) NewDrivePager(
 func (h baseSiteHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {

src/internal/m365/collection/drive/user_drive_handler.go

@@ -42,7 +42,7 @@ func (h baseUserDriveHandler) NewDrivePager(
 func (h baseUserDriveHandler) AugmentItemInfo(
 	dii details.ItemInfo,
 	resource idname.Provider,
-	item models.DriveItemable,
+	item CorsoDriveItemable,
 	size int64,
 	parentPath *path.Builder,
 ) details.ItemInfo {