diff --git a/src/.golangci.yml b/src/.golangci.yml index eaa0e359a..3b57a592d 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -6,11 +6,11 @@ linters: - gci - gofmt - gofumpt + - errcheck - lll - misspell - revive - wsl - - errcheck linters-settings: gci: @@ -80,6 +80,3 @@ issues: linters: - revive text: "import-shadowing:.*'suite' shadows" - # Temporarily skip linting wsl on `connector` package until fixes are merged. - - path: internal/connector/onedrive - linters: wsl diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index b57a80beb..85263ff32 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -64,6 +64,7 @@ func NewCollection( } // Allows tests to set a mock populator c.itemReader = driveItemReader + return c } @@ -105,16 +106,21 @@ func (od *Item) Info() details.ItemInfo { // populateItems iterates through items added to the collection // and uses the collection `itemReader` to read the item func (oc *Collection) populateItems(ctx context.Context) { - var errs error - itemsRead := 0 + var ( + errs error + itemsRead = 0 + ) + for _, itemID := range oc.driveItemIDs { // Read the item itemName, itemData, err := oc.itemReader(ctx, oc.service, oc.driveID, itemID) if err != nil { errs = support.WrapAndAppendf(itemID, err, errs) + if oc.service.ErrPolicy() { break } + continue } // Item read successfully, add to collection @@ -129,7 +135,9 @@ func (oc *Collection) populateItems(ctx context.Context) { }, } } + close(oc.data) + status := support.CreateStatus(ctx, support.Backup, len(oc.driveItemIDs), // items to read itemsRead, // items read successfully diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index ff9eb6138..203af0c6a 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -49,6 +49,7 @@ func (suite *OneDriveCollectionSuite) testStatusUpdater( return func(s *support.ConnectorOperationStatus) { suite.T().Logf("Update status %v, count %d, success %d", s, s.ObjectCount, s.Successful) *statusToUpdate = *s + wg.Done() } } @@ -78,11 +79,15 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() { // Read items from the collection wg.Add(1) + readItems := []data.Stream{} + for item := range coll.Items() { readItems = append(readItems, item) } + wg.Wait() + // Expect only 1 item require.Len(t, readItems, 1) require.Equal(t, 1, collStatus.ObjectCount) @@ -105,8 +110,8 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() { func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() { t := suite.T() - wg := sync.WaitGroup{} collStatus := support.ConnectorOperationStatus{} + wg := sync.WaitGroup{} wg.Add(1) folderPath, err := getCanonicalPath("folderPath", "a-tenant", "a-user") diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 398051078..45b2a5f58 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -90,10 +90,12 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite if err != nil { return err } + if item.GetRoot() != nil { // Skip the root item continue } + if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil { return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName()) } @@ -116,6 +118,7 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite c.statusUpdater, ) } + switch { case item.GetFolder() != nil, item.GetPackage() != nil: // For folders and packages we also create a collection to represent those @@ -148,6 +151,7 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite return errors.Errorf("item type not supported. item name : %s", *item.GetName()) } } + return nil } @@ -162,6 +166,8 @@ func (c *Collections) stats(item models.DriveItemable) error { default: return errors.Errorf("item type not supported. item name : %s", *item.GetName()) } + c.numItems++ + return nil } diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index b8d4c94b2..91a2c0ce8 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -123,6 +123,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedPackageCount: 1, }, } + for _, tt := range tests { suite.T().Run(tt.testCase, func(t *testing.T) { c := NewCollections(tenant, user, &MockGraphService{}, nil) @@ -157,6 +158,7 @@ func driveItem(name string, path string, isFile, isFolder, isPackage bool) model case isPackage: item.SetPackage(models.NewPackage_escaped()) } + return item } diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index d4b41316a..045a0c83d 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -30,6 +30,7 @@ func drives(ctx context.Context, service graph.Service, user string) ([]models.D return nil, errors.Wrapf(err, "failed to retrieve user drives. user: %s, details: %s", user, support.ConnectorStackErrorTrace(err)) } + logger.Ctx(ctx).Debugf("Found %d drives for user %s", len(r.GetValue()), user) return r.GetValue(), nil @@ -50,6 +51,7 @@ func collectItems( // https://docs.microsoft.com/en-us/graph/api/driveitem-delta? // view=graph-rest-1.0&tabs=http#example-4-retrieving-delta-results-using-a-timestamp builder := service.Client().DrivesById(driveID).Root().Delta() + for { r, err := builder.Get() if err != nil { @@ -69,10 +71,12 @@ func collectItems( if _, found := r.GetAdditionalData()[nextLinkKey]; !found { break } + nextLink := r.GetAdditionalData()[nextLinkKey].(*string) logger.Ctx(ctx).Debugf("Found %s nextLink", *nextLink) builder = delta.NewDeltaRequestBuilder(*nextLink, service.Adapter()) } + return nil } diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index 0a8859026..726a37688 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -44,6 +44,7 @@ func driveItemReader( return "", nil, errors.Errorf("file does not have a download URL. ID: %s, %#v", itemID, item.GetAdditionalData()) } + downloadURL := item.GetAdditionalData()[downloadURLKey].(*string) // TODO: We should use the `msgraphgocore` http client which has the right diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 2a08460af..07f9fc054 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -41,6 +41,7 @@ func TestItemIntegrationSuite(t *testing.T) { ); err != nil { t.Skip(err) } + suite.Run(t, new(ItemIntegrationSuite)) } @@ -85,6 +86,7 @@ func (suite *ItemIntegrationSuite) TestItemReader() { break } } + return nil } err = collectItems(ctx, suite, driveID, itemCollector) @@ -104,6 +106,7 @@ func (suite *ItemIntegrationSuite) TestItemReader() { name, itemData, err := driveItemReader(ctx, suite, driveID, driveItemID) require.NoError(suite.T(), err) require.NotEmpty(suite.T(), name) + size, err := io.Copy(io.Discard, itemData) require.NoError(suite.T(), err) require.NotZero(suite.T(), size) diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 394a9550e..e61a2b731 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -56,7 +56,6 @@ func RestoreCollections(ctx context.Context, service graph.Service, dcs []data.C // Iterate through the data collections and restore the contents of each for _, dc := range dcs { - directory := dc.FullPath() drivePath, err := toOneDrivePath(directory) @@ -86,6 +85,7 @@ func RestoreCollections(ctx context.Context, service graph.Service, dcs []data.C // Restore items from the collection exit := false items := dc.Items() + for !exit { select { case <-ctx.Done(): @@ -123,6 +123,7 @@ func createRestoreFolders(ctx context.Context, service graph.Service, driveID st support.ConnectorStackErrorTrace(err), ) } + logger.Ctx(ctx).Debugf("Found Root for Drive %s with ID %s", driveID, *driveRoot.GetId()) parentFolderID := *driveRoot.GetId() @@ -131,8 +132,10 @@ func createRestoreFolders(ctx context.Context, service graph.Service, driveID st if err == nil { parentFolderID = *folderItem.GetId() logger.Ctx(ctx).Debugf("Found %s with ID %s", folder, parentFolderID) + continue } + if err != errFolderNotFound { return "", errors.Wrapf(err, "folder %s not found in drive(%s) parentFolder(%s)", folder, driveID, parentFolderID) }