Update to /connector/sharepoint/data_collections.go

Removal of metadata from SharePoint Library collect call.
Stub left in overall DataCollections for future use.

src/internal/connector/data_collections.go

@@ -95,6 +95,7 @@ func (gc *GraphConnector) DataCollections(
 			ctx,
 			gc.itemClient,
 			sels,
+			metadata,
 			gc.credentials,
 			gc.Service,
 			gc,

src/internal/connector/data_collections_test.go

@@ -256,6 +256,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
 				ctx,
 				graph.HTTPClient(graph.NoTimeout()),
 				test.getSelector(),
+				nil,
 				connector.credentials,
 				connector.Service,
 				connector,

src/internal/connector/onedrive/collection.go

@@ -123,6 +123,7 @@ func NewCollection(
 	switch source {
 	case SharePointSource:
 		c.itemReader = sharePointItemReader
+		c.itemMetaReader = oneDriveItemMetaReader
 	default:
 		c.itemReader = oneDriveItemReader
 		c.itemMetaReader = oneDriveItemMetaReader
@@ -286,22 +287,20 @@ func (oc *Collection) populateItems(ctx context.Context) {
 				metaSuffix = DirMetaFileSuffix
 			}
 
-			if oc.source == OneDriveSource {
+			// Fetch metadata for the file
-				// Fetch metadata for the file
+			if !oc.ctrl.ToggleFeatures.EnablePermissionsBackup {
-				if !oc.ctrl.ToggleFeatures.EnablePermissionsBackup {
+				// We are still writing the metadata file but with
-					// We are still writing the metadata file but with
+				// empty permissions as we don't have a way to
-					// empty permissions as we don't have a way to
+				// signify that the permissions was explicitly
-					// signify that the permissions was explicitly
+				// not added.
-					// not added.
+				itemMeta = io.NopCloser(strings.NewReader("{}"))
-					itemMeta = io.NopCloser(strings.NewReader("{}"))
+				itemMetaSize = 2
-					itemMetaSize = 2
+			} else {
-				} else {
+				itemMeta, itemMetaSize, err = oc.itemMetaReader(ctx, oc.service, oc.driveID, item)
-					itemMeta, itemMetaSize, err = oc.itemMetaReader(ctx, oc.service, oc.driveID, item)
 
-					if err != nil {
+				if err != nil {
-						errUpdater(*item.GetId(), errors.Wrap(err, "failed to get item permissions"))
+					errUpdater(*item.GetId(), errors.Wrap(err, "failed to get item permissions"))
-						return
+					return
-					}
 				}
 			}
 
@@ -315,10 +314,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
 			}
 
 			if isFile {
-				dataSuffix := ""
+				dataSuffix := DataFileSuffix
-				if oc.source == OneDriveSource {
-					dataSuffix = DataFileSuffix
-				}
 
 				// Construct a new lazy readCloser to feed to the collection consumer.
 				// This ensures that downloads won't be attempted unless that consumer
@@ -371,39 +367,16 @@ func (oc *Collection) populateItems(ctx context.Context) {
 				}
 			}
 
+			var metaItem *Item
 			if oc.source == OneDriveSource {
-				metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) {
+				metaItem = handleOneDriveMeta(ctx, itemMeta, itemInfo.OneDrive,
-					progReader, closer := observe.ItemProgress(
+					itemName, metaSuffix, itemMetaSize)
-						ctx, itemMeta, observe.ItemBackupMsg,
+			} else {
-						observe.PII(itemName+metaSuffix), int64(itemMetaSize))
+				metaItem = handleSharePointMeta(ctx, itemMeta, itemInfo.SharePoint, itemName, metaSuffix, itemMetaSize)
-					go closer()
-					return progReader, nil
-				})
-
-				// TODO(meain): Remove this once we change to always
-				// backing up permissions. Until then we cannot rely
-				// on weather the previous data is what we need as the
-				// user might have not backup up permissions in the
-				// previous run.
-				metaItemInfo := details.ItemInfo{}
-				metaItemInfo.OneDrive = &details.OneDriveInfo{
-					Created:    itemInfo.OneDrive.Created,
-					ItemName:   itemInfo.OneDrive.ItemName,
-					DriveName:  itemInfo.OneDrive.DriveName,
-					ItemType:   itemInfo.OneDrive.ItemType,
-					Modified:   time.Now(), // set to current time to always refresh
-					Owner:      itemInfo.OneDrive.Owner,
-					ParentPath: itemInfo.OneDrive.ParentPath,
-					Size:       itemInfo.OneDrive.Size,
-				}
-
-				oc.data <- &Item{
-					id:   itemName + metaSuffix,
-					data: metaReader,
-					info: metaItemInfo,
-				}
 			}
 
+			oc.data <- metaItem
+
 			// Item read successfully, add to collection
 			if isFile {
 				atomic.AddInt64(&itemsRead, 1)
@@ -439,3 +412,93 @@ func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRe
 	logger.Ctx(ctx).Debugw("done streaming items", "status", status.String())
 	oc.statusUpdater(status)
 }
+
+func createMetaOneDrive(info *details.OneDriveInfo) details.ItemInfo {
+	// TODO(meain): Remove this once we change to always
+	// backing up permissions. Until then we cannot rely
+	// on weather the previous data is what we need as the
+	// user might have not backup up permissions in the
+	// previous run.
+	metaItemInfo := details.ItemInfo{}
+	metaItemInfo.OneDrive = &details.OneDriveInfo{
+		Created:    info.Created,
+		ItemName:   info.ItemName,
+		DriveName:  info.DriveName,
+		ItemType:   info.ItemType,
+		Modified:   time.Now(), // set to current time to always refresh
+		Owner:      info.Owner,
+		ParentPath: info.ParentPath,
+		Size:       info.Size,
+	}
+
+	return metaItemInfo
+}
+
+func createMetaSharePoint(info *details.SharePointInfo) details.ItemInfo {
+	metaItemInfo := details.ItemInfo{}
+	metaItemInfo.SharePoint = &details.SharePointInfo{
+		Created:    info.Created,
+		ItemName:   info.ItemName,
+		DriveName:  info.DriveName,
+		ItemType:   info.ItemType,
+		Modified:   time.Now(),
+		Owner:      info.Owner,
+		ParentPath: info.ParentPath,
+		WebURL:     info.WebURL,
+		Size:       info.Size,
+	}
+
+	return metaItemInfo
+}
+
+func handleOneDriveMeta(
+	ctx context.Context,
+	itemMeta io.ReadCloser,
+	info *details.OneDriveInfo,
+	name, suffix string,
+	size int,
+) *Item {
+	metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) {
+		progReader, closer := observe.ItemProgress(
+			ctx, itemMeta, observe.ItemBackupMsg,
+			observe.PII(name+suffix), int64(size))
+		go closer()
+		return progReader, nil
+	})
+
+	metaItemInfo := createMetaOneDrive(info)
+
+	item := &Item{
+		id:   name + suffix,
+		data: metaReader,
+		info: metaItemInfo,
+	}
+
+	return item
+}
+
+func handleSharePointMeta(
+	ctx context.Context,
+	itemMeta io.ReadCloser,
+	info *details.SharePointInfo,
+	name, suffix string,
+	size int,
+) *Item {
+	metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) {
+		progReader, closer := observe.ItemProgress(
+			ctx, itemMeta, observe.ItemBackupMsg,
+			observe.PII(name+suffix), int64(size))
+		go closer()
+		return progReader, nil
+	})
+
+	metaItemInfo := createMetaSharePoint(info)
+
+	item := &Item{
+		id:   name + suffix,
+		data: metaReader,
+		info: metaItemInfo,
+	}
+
+	return item
+}

src/internal/connector/onedrive/collection_test.go

@@ -210,11 +210,9 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
 
 			wg.Wait()
 
-			if test.source == OneDriveSource {
+			// SharePoint and OneDrive contain
-				require.Len(t, readItems, 2) // .data and .meta
+			// .data and .meta files
-			} else {
+			require.Len(t, readItems, 2) // .data and .meta
-				require.Len(t, readItems, 1)
-			}
 
 			// Expect only 1 item
 			require.Equal(t, 1, collStatus.ObjectCount)
@@ -224,11 +222,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
 			readItem := readItems[0]
 			readItemInfo := readItem.(data.StreamInfo)
 
-			if test.source == OneDriveSource {
+			assert.Equal(t, testItemName+DataFileSuffix, readItem.UUID())
-				assert.Equal(t, testItemName+DataFileSuffix, readItem.UUID())
-			} else {
-				assert.Equal(t, testItemName, readItem.UUID())
-			}
 
 			require.Implements(t, (*data.StreamModTime)(nil), readItem)
 			mt := readItem.(data.StreamModTime)

src/internal/connector/sharepoint/data_collections.go

@@ -30,6 +30,7 @@ func DataCollections(
 	ctx context.Context,
 	itemClient *http.Client,
 	selector selectors.Selector,
+	metadata []data.RestoreCollection,
 	creds account.M365Config,
 	serv graph.Servicer,
 	su statusUpdater,
@@ -156,7 +157,8 @@ func collectLibraries(
 
 	logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Library collections")
 
-	colls := onedrive.NewCollections(
+	// TODO (ashmrtn): metadata for incremental support
+	odcs, excludes, err := onedrive.NewCollections(
 		itemClient,
 		tenantID,
 		siteID,
@@ -164,11 +166,7 @@ func collectLibraries(
 		folderMatcher{scope},
 		serv,
 		updater.UpdateStatus,
-		ctrlOpts)
+		ctrlOpts).Get(ctx, nil)
-
-	// TODO(ashmrtn): Pass previous backup metadata when SharePoint supports delta
-	// token-based incrementals.
-	odcs, excludes, err := colls.Get(ctx, nil)
 	if err != nil {
 		return nil, nil, support.WrapAndAppend(siteID, err, errs)
 	}