From dba6c2bf2c9187e2057e5f861cabc0a086ca34ef Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Thu, 20 Jul 2023 13:19:03 +0530 Subject: [PATCH 01/10] Update CHANGELOG for 0.11.1 (#3863) --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7fdca0d..44a5075af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +## [v0.11.1] (beta) - 2023-07-20 + ### Fixed - Allow repo connect to succeed when a `corso.toml` file was not provided but configuration is specified using environment variables and flags. @@ -328,7 +330,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Miscellaneous - Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35)) -[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.0...HEAD +[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.1...HEAD +[v0.11.1]: https://github.com/alcionai/corso/compare/v0.11.0...v0.11.1 [v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0 [v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0 [v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0 From e130f85da334c138ba39decae767e5197758ce55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Jul 2023 18:45:19 +0000 Subject: [PATCH 02/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.302=20to=201.44.304=20in=20/src=20(#?= =?UTF-8?q?3862)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.302 to 1.44.304.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.304 (2023-07-20)

Service Client Updates

  • service/savingsplans: Updates service documentation

Release v1.44.303 (2023-07-19)

Service Client Updates

  • service/cloudformation: Updates service API and documentation
    • SDK and documentation updates for GetTemplateSummary API (unrecognized resources)
  • service/ec2: Updates service documentation
    • Amazon EC2 documentation updates.
  • service/grafana: Updates service API, documentation, and paginators
  • service/medical-imaging: Adds new service
  • service/ram: Updates service API and documentation
  • service/ssm-sap: Updates service API and documentation
  • service/wafv2: Updates service API and documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.302&new-version=1.44.304)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 99581228f..6ebbb2adb 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.302 + github.com/aws/aws-sdk-go v1.44.304 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 030dabe8a..a66eb73f4 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.302 h1:ST3ko6GrJKn3Xi+nAvxjG3uk/V1pW8KC52WLeIxqqNk= -github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.304 h1:crcJBVeewWcVAXDQChzJWZYGFq9i1TYLycAtQ6Xpi4c= +github.com/aws/aws-sdk-go v1.44.304/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 9359679f9967b7f0eaae245d89c96f2a3727f79a Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Jul 2023 15:05:30 -0600 Subject: [PATCH 03/10] add api funcs for creating documentLibs (#3793) Adds api handlers for creating document libraries in sharepoint. This is the first step in allowing us to restore drives that were deleted between backup and restore. --- #### Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 4 + src/internal/common/idname/idname.go | 33 +- src/internal/data/mock/collection.go | 48 ++ src/internal/m365/controller.go | 20 + src/internal/m365/controller_test.go | 90 +++- src/internal/m365/mock/connector.go | 2 + src/internal/m365/onedrive/handlers.go | 15 +- .../m365/onedrive/item_collector_test.go | 4 +- src/internal/m365/onedrive/item_handler.go | 14 + src/internal/m365/onedrive/mock/handlers.go | 16 + src/internal/m365/onedrive/restore.go | 214 ++++++++- src/internal/m365/onedrive/restore_test.go | 436 +++++++++++++++++- src/internal/m365/restore.go | 6 + .../m365/sharepoint/library_handler.go | 36 +- src/internal/m365/sharepoint/restore.go | 19 +- src/internal/operations/inject/inject.go | 11 + src/internal/operations/restore.go | 14 +- src/pkg/control/restore.go | 9 +- src/pkg/services/m365/api/lists.go | 64 +++ src/pkg/services/m365/api/lists_test.go | 57 +++ website/docs/support/known-issues.md | 2 - 21 files changed, 1051 insertions(+), 63 deletions(-) create mode 100644 src/pkg/services/m365/api/lists.go create mode 100644 src/pkg/services/m365/api/lists_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a5075af..296b01a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +### Fixed +- SharePoint document libraries deleted after the last backup can now be restored. + ## [v0.11.1] (beta) - 2023-07-20 ### Fixed @@ -23,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Return a ServiceNotEnabled error when a tenant has no active SharePoint license. - Added retries for http/2 stream connection failures when downloading large item content. +- SharePoint document libraries that were deleted after the last backup can now be restored. ### Known issues - If a link share is created for an item with inheritance disabled diff --git a/src/internal/common/idname/idname.go b/src/internal/common/idname/idname.go index d56fab025..0367d954b 100644 --- a/src/internal/common/idname/idname.go +++ b/src/internal/common/idname/idname.go @@ -40,6 +40,11 @@ type Cacher interface { ProviderForName(id string) Provider } +type CacheBuilder interface { + Add(id, name string) + Cacher +} + var _ Cacher = &cache{} type cache struct { @@ -47,17 +52,29 @@ type cache struct { nameToID map[string]string } -func NewCache(idToName map[string]string) cache { - nti := make(map[string]string, len(idToName)) - - for id, name := range idToName { - nti[name] = id +func NewCache(idToName map[string]string) *cache { + c := cache{ + idToName: map[string]string{}, + nameToID: map[string]string{}, } - return cache{ - idToName: idToName, - nameToID: nti, + if len(idToName) > 0 { + nti := make(map[string]string, len(idToName)) + + for id, name := range idToName { + nti[name] = id + } + + c.idToName = idToName + c.nameToID = nti } + + return &c +} + +func (c *cache) Add(id, name string) { + c.idToName[strings.ToLower(id)] = name + c.nameToID[strings.ToLower(name)] = id } // IDOf returns the id associated with the given name. diff --git a/src/internal/data/mock/collection.go b/src/internal/data/mock/collection.go index 63f2b2dd8..55f291a7f 100644 --- a/src/internal/data/mock/collection.go +++ b/src/internal/data/mock/collection.go @@ -1,12 +1,24 @@ package mock import ( + "context" "io" "time" + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" ) +// --------------------------------------------------------------------------- +// stream +// --------------------------------------------------------------------------- + +var _ data.Stream = &Stream{} + type Stream struct { ID string Reader io.ReadCloser @@ -52,3 +64,39 @@ type errReader struct { func (er errReader) Read([]byte) (int, error) { return 0, er.readErr } + +// --------------------------------------------------------------------------- +// collection +// --------------------------------------------------------------------------- + +var ( + _ data.Collection = &Collection{} + _ data.BackupCollection = &Collection{} + _ data.RestoreCollection = &Collection{} +) + +type Collection struct{} + +func (c Collection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Stream { + return nil +} + +func (c Collection) FullPath() path.Path { + return nil +} + +func (c Collection) PreviousPath() path.Path { + return nil +} + +func (c Collection) State() data.CollectionState { + return data.NewState +} + +func (c Collection) DoNotMergeItems() bool { + return true +} + +func (c Collection) FetchItemByName(ctx context.Context, name string) (data.Stream, error) { + return &Stream{}, clues.New("not implemented") +} diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index b0c8792e5..9b037350b 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -47,6 +48,11 @@ type Controller struct { // mutex used to synchronize updates to `status` mu sync.Mutex status support.ControllerOperationStatus // contains the status of the last run status + + // backupDriveIDNames is populated on restore. It maps the backup's + // drive names to their id. Primarily for use when creating or looking + // up a new drive. + backupDriveIDNames idname.CacheBuilder } func NewController( @@ -142,6 +148,20 @@ func (ctrl *Controller) incrementAwaitingMessages() { ctrl.wg.Add(1) } +func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { + if ctrl.backupDriveIDNames == nil { + ctrl.backupDriveIDNames = idname.NewCache(map[string]string{}) + } + + if dii.SharePoint != nil { + ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) + } + + if dii.OneDrive != nil { + ctrl.backupDriveIDNames.Add(dii.OneDrive.DriveID, dii.OneDrive.DriveName) + } +} + // --------------------------------------------------------------------------- // Resource Lookup Handling // --------------------------------------------------------------------------- diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 6d04b7e9e..ef729493b 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -12,8 +12,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" + dataMock "github.com/alcionai/corso/src/internal/data/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" @@ -22,6 +24,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" @@ -260,6 +263,82 @@ func (suite *ControllerUnitSuite) TestController_Wait() { assert.Equal(t, int64(4), result.Bytes) } +func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { + var ( + odid = "od-id" + odname = "od-name" + spid = "sp-id" + spname = "sp-name" + // intentionally declared outside the test loop + ctrl = &Controller{ + wg: &sync.WaitGroup{}, + region: &trace.Region{}, + backupDriveIDNames: idname.NewCache(nil), + } + ) + + table := []struct { + name string + service path.ServiceType + cat path.CategoryType + dii details.ItemInfo + expectID string + expectName string + }{ + { + name: "exchange", + dii: details.ItemInfo{ + Exchange: &details.ExchangeInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "folder", + dii: details.ItemInfo{ + Folder: &details.FolderInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "onedrive", + dii: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + DriveID: odid, + DriveName: odname, + }, + }, + expectID: odid, + expectName: odname, + }, + { + name: "sharepoint", + dii: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + DriveID: spid, + DriveName: spname, + }, + }, + expectID: spid, + expectName: spname, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctrl.CacheItemInfo(test.dii) + + name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID) + assert.Equal(t, test.expectName, name) + + id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName) + assert.Equal(t, test.expectID, id) + }) + } +} + // --------------------------------------------------------------------------- // Integration tests // --------------------------------------------------------------------------- @@ -315,7 +394,7 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { RestorePermissions: true, ToggleFeatures: control.Toggles{}, }, - nil, + []data.RestoreCollection{&dataMock.Collection{}}, fault.New(true), count.New()) assert.Error(t, err, clues.ToCore(err)) @@ -397,13 +476,8 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { test.col, fault.New(true), count.New()) - require.NoError(t, err, clues.ToCore(err)) - assert.NotNil(t, deets) - - stats := suite.ctrl.Wait() - assert.Zero(t, stats.Objects) - assert.Zero(t, stats.Folders) - assert.Zero(t, stats.Successes) + require.Error(t, err, clues.ToCore(err)) + assert.Nil(t, deets) }) } } diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 05cb8e159..977306883 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -69,3 +69,5 @@ func (ctrl Controller) ConsumeRestoreCollections( ) (*details.Details, error) { return ctrl.Deets, ctrl.Err } + +func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} diff --git a/src/internal/m365/onedrive/handlers.go b/src/internal/m365/onedrive/handlers.go index dfea5ee17..cb33b373d 100644 --- a/src/internal/m365/onedrive/handlers.go +++ b/src/internal/m365/onedrive/handlers.go @@ -35,6 +35,7 @@ type BackupHandler interface { api.Getter GetItemPermissioner GetItemer + NewDrivePagerer // PathPrefix constructs the service and category specific path prefix for // the given values. @@ -49,7 +50,6 @@ type BackupHandler interface { // ServiceCat returns the service and category used by this implementation. ServiceCat() (path.ServiceType, path.CategoryType) - NewDrivePager(resourceOwner string, fields []string) api.DrivePager NewItemPager(driveID, link string, fields []string) api.DriveItemDeltaEnumerator // FormatDisplayPath creates a human-readable string to represent the // provided path. @@ -61,6 +61,10 @@ type BackupHandler interface { IncludesDir(dir string) bool } +type NewDrivePagerer interface { + NewDrivePager(resourceOwner string, fields []string) api.DrivePager +} + type GetItemPermissioner interface { GetItemPermission( ctx context.Context, @@ -86,7 +90,9 @@ type RestoreHandler interface { GetItemsByCollisionKeyser GetRootFolderer ItemInfoAugmenter + NewDrivePagerer NewItemContentUploader + PostDriver PostItemInContainerer DeleteItemPermissioner UpdateItemPermissioner @@ -145,6 +151,13 @@ type UpdateItemLinkSharer interface { ) (models.Permissionable, error) } +type PostDriver interface { + PostDrive( + ctx context.Context, + protectedResourceID, driveName string, + ) (models.Driveable, error) +} + type PostItemInContainerer interface { PostItemInContainer( ctx context.Context, diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index 6e4a79be6..ec2ab26af 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -361,8 +361,8 @@ func (suite *OneDriveIntgSuite) TestCreateGetDeleteFolder() { Folders: folderElements, } - caches := NewRestoreCaches() - caches.DriveIDToRootFolderID[driveID] = ptr.Val(rootFolder.GetId()) + caches := NewRestoreCaches(nil) + caches.DriveIDToDriveInfo[driveID] = driveInfo{rootFolderID: ptr.Val(rootFolder.GetId())} rh := NewRestoreHandler(suite.ac) diff --git a/src/internal/m365/onedrive/item_handler.go b/src/internal/m365/onedrive/item_handler.go index 0b1420cf0..64701da8f 100644 --- a/src/internal/m365/onedrive/item_handler.go +++ b/src/internal/m365/onedrive/item_handler.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -133,6 +134,19 @@ func NewRestoreHandler(ac api.Client) *itemRestoreHandler { return &itemRestoreHandler{ac.Drives()} } +func (h itemRestoreHandler) PostDrive( + context.Context, + string, string, +) (models.Driveable, error) { + return nil, clues.New("creating drives in oneDrive is not supported") +} + +func (h itemRestoreHandler) NewDrivePager( + resourceOwner string, fields []string, +) api.DrivePager { + return h.ac.NewUserDrivePager(resourceOwner, fields) +} + // AugmentItemInfo will populate a details.OneDriveInfo struct // with properties from the drive item. ItemSize is specified // separately for restore processes because the local itemable diff --git a/src/internal/m365/onedrive/mock/handlers.go b/src/internal/m365/onedrive/mock/handlers.go index 92b4573e6..75dd3c3f1 100644 --- a/src/internal/m365/onedrive/mock/handlers.go +++ b/src/internal/m365/onedrive/mock/handlers.go @@ -249,9 +249,25 @@ type RestoreHandler struct { PostItemResp models.DriveItemable PostItemErr error + DrivePagerV api.DrivePager + + PostDriveResp models.Driveable + PostDriveErr error + UploadSessionErr error } +func (h RestoreHandler) PostDrive( + ctx context.Context, + protectedResourceID, driveName string, +) (models.Driveable, error) { + return h.PostDriveResp, h.PostDriveErr +} + +func (h RestoreHandler) NewDrivePager(string, []string) api.DrivePager { + return h.DrivePagerV +} + func (h *RestoreHandler) AugmentItemInfo( details.ItemInfo, models.DriveItemable, diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 84b8f1cd0..f951419be 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -15,6 +15,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -37,9 +38,17 @@ const ( maxUploadRetries = 3 ) +type driveInfo struct { + id string + name string + rootFolderID string +} + type restoreCaches struct { + BackupDriveIDName idname.Cacher collisionKeyToItemID map[string]api.DriveItemIDType - DriveIDToRootFolderID map[string]string + DriveIDToDriveInfo map[string]driveInfo + DriveNameToDriveInfo map[string]driveInfo Folders *folderCache OldLinkShareIDToNewID map[string]string OldPermIDToNewID map[string]string @@ -48,10 +57,74 @@ type restoreCaches struct { pool sync.Pool } -func NewRestoreCaches() *restoreCaches { +func (rc *restoreCaches) AddDrive( + ctx context.Context, + md models.Driveable, + grf GetRootFolderer, +) error { + di := driveInfo{ + id: ptr.Val(md.GetId()), + name: ptr.Val(md.GetName()), + } + + ctx = clues.Add(ctx, "drive_info", di) + + root, err := grf.GetRootFolder(ctx, di.id) + if err != nil { + return clues.Wrap(err, "getting drive root id") + } + + di.rootFolderID = ptr.Val(root.GetId()) + + rc.DriveIDToDriveInfo[di.id] = di + rc.DriveNameToDriveInfo[di.name] = di + + return nil +} + +// Populate looks up drive items available to the protectedResource +// and adds their info to the caches. +func (rc *restoreCaches) Populate( + ctx context.Context, + gdparf GetDrivePagerAndRootFolderer, + protectedResourceID string, +) error { + drives, err := api.GetAllDrives( + ctx, + gdparf.NewDrivePager(protectedResourceID, nil), + true, + maxDrivesRetries) + if err != nil { + return clues.Wrap(err, "getting drives") + } + + for _, md := range drives { + if err := rc.AddDrive(ctx, md, gdparf); err != nil { + return clues.Wrap(err, "caching drive") + } + } + + return nil +} + +type GetDrivePagerAndRootFolderer interface { + GetRootFolderer + NewDrivePagerer +} + +func NewRestoreCaches( + backupDriveIDNames idname.Cacher, +) *restoreCaches { + // avoid nil panics + if backupDriveIDNames == nil { + backupDriveIDNames = idname.NewCache(nil) + } + return &restoreCaches{ + BackupDriveIDName: backupDriveIDNames, collisionKeyToItemID: map[string]api.DriveItemIDType{}, - DriveIDToRootFolderID: map[string]string{}, + DriveIDToDriveInfo: map[string]driveInfo{}, + DriveNameToDriveInfo: map[string]driveInfo{}, Folders: NewFolderCache(), OldLinkShareIDToNewID: map[string]string{}, OldPermIDToNewID: map[string]string{}, @@ -73,19 +146,27 @@ func ConsumeRestoreCollections( backupVersion int, restoreCfg control.RestoreConfig, opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - caches = NewRestoreCaches() - el = errs.Local() + restoreMetrics support.CollectionMetrics + el = errs.Local() + caches = NewRestoreCaches(backupDriveIDNames) + protectedResourceID = dcs[0].FullPath().ResourceOwner() + fallbackDriveName = restoreCfg.Location ) ctx = clues.Add(ctx, "backup_version", backupVersion) + err := caches.Populate(ctx, rh, protectedResourceID) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. data.SortRestoreCollections(dcs) @@ -102,7 +183,7 @@ func ConsumeRestoreCollections( ictx = clues.Add( ctx, "category", dc.FullPath().Category(), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owner", clues.Hide(protectedResourceID), "full_path", dc.FullPath()) ) @@ -115,6 +196,7 @@ func ConsumeRestoreCollections( caches, deets, opts.RestorePermissions, + fallbackDriveName, errs, ctr.Local()) if err != nil { @@ -152,18 +234,20 @@ func RestoreCollection( caches *restoreCaches, deets *details.Builder, restorePerms bool, // TODD: move into restoreConfig + fallbackDriveName string, errs *fault.Bus, ctr *count.Bus, ) (support.CollectionMetrics, error) { var ( - metrics = support.CollectionMetrics{} - directory = dc.FullPath() - el = errs.Local() - metricsObjects int64 - metricsBytes int64 - metricsSuccess int64 - wg sync.WaitGroup - complete bool + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + protectedResourceID = directory.ResourceOwner() + el = errs.Local() + metricsObjects int64 + metricsBytes int64 + metricsSuccess int64 + wg sync.WaitGroup + complete bool ) ctx, end := diagnostics.Span(ctx, "gc:drive:restoreCollection", diagnostics.Label("path", directory)) @@ -174,15 +258,23 @@ func RestoreCollection( return metrics, clues.Wrap(err, "creating drive path").WithClues(ctx) } - if _, ok := caches.DriveIDToRootFolderID[drivePath.DriveID]; !ok { - root, err := rh.GetRootFolder(ctx, drivePath.DriveID) - if err != nil { - return metrics, clues.Wrap(err, "getting drive root id") - } - - caches.DriveIDToRootFolderID[drivePath.DriveID] = ptr.Val(root.GetId()) + di, err := ensureDriveExists( + ctx, + rh, + caches, + drivePath, + protectedResourceID, + fallbackDriveName) + if err != nil { + return metrics, clues.Wrap(err, "ensuring drive exists") } + // clobber the drivePath details with the details retrieved + // in the ensure func, as they might have changed to reflect + // a different drive as a restore location. + drivePath.DriveID = di.id + drivePath.Root = di.rootFolderID + // Assemble folder hierarchy we're going to restore into (we recreate the folder hierarchy // from the backup under this the restore folder instead of root) // i.e. Restore into `/` @@ -704,7 +796,7 @@ func createRestoreFolders( driveID = drivePath.DriveID folders = restoreDir.Elements() location = path.Builder{}.Append(driveID) - parentFolderID = caches.DriveIDToRootFolderID[drivePath.DriveID] + parentFolderID = caches.DriveIDToDriveInfo[drivePath.DriveID].rootFolderID ) ctx = clues.Add( @@ -1113,3 +1205,79 @@ func AugmentRestorePaths( return paths, nil } + +type PostDriveAndGetRootFolderer interface { + PostDriver + GetRootFolderer +} + +// ensureDriveExists looks up the drive by its id. If no drive is found with +// that ID, a new drive is generated with the same name. If the name collides +// with an existing drive, a number is appended to the drive name. Eg: foo -> +// foo 1. This will repeat as many times as is needed. +// Returns the root folder of the drive +func ensureDriveExists( + ctx context.Context, + pdagrf PostDriveAndGetRootFolderer, + caches *restoreCaches, + drivePath *path.DrivePath, + protectedResourceID, fallbackDriveName string, +) (driveInfo, error) { + driveID := drivePath.DriveID + + // the drive might already be cached by ID. it's okay + // if the name has changed. the ID is a better reference + // anyway. + if di, ok := caches.DriveIDToDriveInfo[driveID]; ok { + return di, nil + } + + var ( + newDriveName = fallbackDriveName + newDrive models.Driveable + err error + ) + + // if the drive wasn't found by ID, maybe we can find a + // drive with the same name but different ID. + // start by looking up the old drive's name + oldName, ok := caches.BackupDriveIDName.NameOf(driveID) + if ok { + // check for drives that currently have the same name + if di, ok := caches.DriveNameToDriveInfo[oldName]; ok { + return di, nil + } + + // if no current drives have the same name, we'll make + // a new drive with that name. + newDriveName = oldName + } + + nextDriveName := newDriveName + + // For sharepoint, document libraries can collide by name with + // item types beyond just drive. Lists, for example, cannot share + // names with document libraries (they're the same type, actually). + // In those cases we need to rename the drive until we can create + // one without a collision. + for i := 1; ; i++ { + ictx := clues.Add(ctx, "new_drive_name", clues.Hide(nextDriveName)) + + newDrive, err = pdagrf.PostDrive(ictx, protectedResourceID, nextDriveName) + if err != nil && !errors.Is(err, graph.ErrItemAlreadyExistsConflict) { + return driveInfo{}, clues.Wrap(err, "creating new drive") + } + + if err == nil { + break + } + + nextDriveName = fmt.Sprintf("%s %d", newDriveName, i) + } + + if err := caches.AddDrive(ctx, newDrive, pdagrf); err != nil { + return driveInfo{}, clues.Wrap(err, "adding drive to cache").OrNil() + } + + return caches.DriveIDToDriveInfo[ptr.Val(newDrive.GetId())], nil +} diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index 4128661f5..dbb7317c9 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" @@ -21,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" + apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type RestoreUnitSuite struct { @@ -491,7 +493,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { mndi.SetId(ptr.To(mndiID)) var ( - caches = NewRestoreCaches() + caches = NewRestoreCaches(nil) rh = &mock.RestoreHandler{ PostItemResp: models.NewDriveItem(), DeleteItemErr: test.deleteErr, @@ -617,3 +619,435 @@ func (suite *RestoreUnitSuite) TestCreateFolder() { }) } } + +type mockGRF struct { + err error + rootFolder models.DriveItemable +} + +func (m *mockGRF) GetRootFolder( + context.Context, + string, +) (models.DriveItemable, error) { + return m.rootFolder, m.err +} + +func (suite *RestoreUnitSuite) TestRestoreCaches_AddDrive() { + rfID := "this-is-id" + driveID := "another-id" + name := "name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + table := []struct { + name string + mock *mockGRF + expectErr require.ErrorAssertionFunc + expectID string + checkValues bool + }{ + { + name: "good", + mock: &mockGRF{rootFolder: rf}, + expectErr: require.NoError, + expectID: rfID, + checkValues: true, + }, + { + name: "err", + mock: &mockGRF{err: assert.AnError}, + expectErr: require.Error, + expectID: "", + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := NewRestoreCaches(nil) + err := rc.AddDrive(ctx, md, test.mock) + test.expectErr(t, err, clues.ToCore(err)) + + if test.checkValues { + idResult := rc.DriveIDToDriveInfo[driveID] + assert.Equal(t, driveID, idResult.id, "drive id") + assert.Equal(t, name, idResult.name, "drive name") + assert.Equal(t, test.expectID, idResult.rootFolderID, "root folder id") + + nameResult := rc.DriveNameToDriveInfo[name] + assert.Equal(t, driveID, nameResult.id, "drive id") + assert.Equal(t, name, nameResult.name, "drive name") + assert.Equal(t, test.expectID, nameResult.rootFolderID, "root folder id") + } + }) + } +} + +type mockGDPARF struct { + err error + rootFolder models.DriveItemable + pager *apiMock.DrivePager +} + +func (m *mockGDPARF) GetRootFolder( + context.Context, + string, +) (models.DriveItemable, error) { + return m.rootFolder, m.err +} + +func (m *mockGDPARF) NewDrivePager( + string, + []string, +) api.DrivePager { + return m.pager +} + +func (suite *RestoreUnitSuite) TestRestoreCaches_Populate() { + rfID := "this-is-id" + driveID := "another-id" + name := "name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + table := []struct { + name string + mock *apiMock.DrivePager + expectErr require.ErrorAssertionFunc + expectLen int + checkValues bool + }{ + { + name: "no results", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Drives: []models.Driveable{}}, + }, + }, + expectErr: require.NoError, + expectLen: 0, + }, + { + name: "one result", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Drives: []models.Driveable{md}}, + }, + }, + expectErr: require.NoError, + expectLen: 1, + checkValues: true, + }, + { + name: "error", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Err: assert.AnError}, + }, + }, + expectErr: require.Error, + expectLen: 0, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + gdparf := &mockGDPARF{ + rootFolder: rf, + pager: test.mock, + } + + rc := NewRestoreCaches(nil) + err := rc.Populate(ctx, gdparf, "shmoo") + test.expectErr(t, err, clues.ToCore(err)) + + assert.Len(t, rc.DriveIDToDriveInfo, test.expectLen) + assert.Len(t, rc.DriveNameToDriveInfo, test.expectLen) + + if test.checkValues { + idResult := rc.DriveIDToDriveInfo[driveID] + assert.Equal(t, driveID, idResult.id, "drive id") + assert.Equal(t, name, idResult.name, "drive name") + assert.Equal(t, rfID, idResult.rootFolderID, "root folder id") + + nameResult := rc.DriveNameToDriveInfo[name] + assert.Equal(t, driveID, nameResult.id, "drive id") + assert.Equal(t, name, nameResult.name, "drive name") + assert.Equal(t, rfID, nameResult.rootFolderID, "root folder id") + } + }) + } +} + +type mockPDAGRF struct { + i int + postResp []models.Driveable + postErr []error + + grf mockGRF +} + +func (m *mockPDAGRF) PostDrive( + ctx context.Context, + protectedResourceID, driveName string, +) (models.Driveable, error) { + defer func() { m.i++ }() + + md := m.postResp[m.i] + if md != nil { + md.SetName(&driveName) + } + + return md, m.postErr[m.i] +} + +func (m *mockPDAGRF) GetRootFolder( + ctx context.Context, + driveID string, +) (models.DriveItemable, error) { + return m.grf.rootFolder, m.grf.err +} + +func (suite *RestoreUnitSuite) TestEnsureDriveExists() { + rfID := "this-is-id" + driveID := "another-id" + oldID := "old-id" + name := "name" + otherName := "other name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + grf := mockGRF{rootFolder: rf} + + makeMD := func() models.Driveable { + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + return md + } + + dp := &path.DrivePath{ + DriveID: driveID, + Root: "root:", + Folders: path.Elements{}, + } + + oldDP := &path.DrivePath{ + DriveID: oldID, + Root: "root:", + Folders: path.Elements{}, + } + + populatedCache := func(id string) *restoreCaches { + rc := NewRestoreCaches(nil) + di := driveInfo{ + id: id, + name: name, + } + rc.DriveIDToDriveInfo[id] = di + rc.DriveNameToDriveInfo[name] = di + + return rc + } + + oldDriveIDNames := idname.NewCache(nil) + oldDriveIDNames.Add(oldID, name) + + idSwitchedCache := func() *restoreCaches { + rc := NewRestoreCaches(oldDriveIDNames) + di := driveInfo{ + id: "diff", + name: name, + } + rc.DriveIDToDriveInfo["diff"] = di + rc.DriveNameToDriveInfo[name] = di + + return rc + } + + table := []struct { + name string + dp *path.DrivePath + mock *mockPDAGRF + rc *restoreCaches + expectErr require.ErrorAssertionFunc + fallbackName string + expectName string + expectID string + skipValueChecks bool + }{ + { + name: "drive already in cache", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: populatedCache(driveID), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + { + name: "drive with same name but different id exists", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: idSwitchedCache(), + expectErr: require.NoError, + fallbackName: otherName, + expectName: name, + expectID: "diff", + }, + { + name: "drive created with old name", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: NewRestoreCaches(oldDriveIDNames), + expectErr: require.NoError, + fallbackName: otherName, + expectName: name, + expectID: driveID, + }, + { + name: "drive created with fallback name", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.NoError, + fallbackName: otherName, + expectName: otherName, + expectID: driveID, + }, + { + name: "error creating drive", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil}, + postErr: []error{assert.AnError}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.Error, + fallbackName: name, + expectName: "", + skipValueChecks: true, + expectID: driveID, + }, + { + name: "drive name already exists", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: populatedCache("beaux"), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + { + name: "list with name already exists", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.NoError, + fallbackName: name, + expectName: name + " 1", + expectID: driveID, + }, + { + name: "list with old name already exists", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: NewRestoreCaches(oldDriveIDNames), + expectErr: require.NoError, + fallbackName: name, + expectName: name + " 1", + expectID: driveID, + }, + { + name: "drive and list with name already exist", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: populatedCache(driveID), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := test.rc + + di, err := ensureDriveExists( + ctx, + test.mock, + rc, + test.dp, + "prID", + test.fallbackName) + test.expectErr(t, err, clues.ToCore(err)) + + if !test.skipValueChecks { + assert.Equal(t, test.expectName, di.name, "ensured drive has expected name") + assert.Equal(t, test.expectID, di.id, "ensured drive has expected id") + + nameResult := rc.DriveNameToDriveInfo[test.expectName] + assert.Equal(t, test.expectName, nameResult.name, "found drive entry with expected name") + } + }) + } +} diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 3c5e3e646..31e36e2bb 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -38,6 +38,10 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) ctx = clues.Add(ctx, "restore_config", restoreCfg) // TODO(rkeepers): needs PII control + if len(dcs) == 0 { + return nil, clues.New("no data collections to restore") + } + var ( status *support.ControllerOperationStatus deets = &details.Builder{} @@ -54,6 +58,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( backupVersion, restoreCfg, opts, + ctrl.backupDriveIDNames, dcs, deets, errs, @@ -65,6 +70,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctrl.AC, restoreCfg, opts, + ctrl.backupDriveIDNames, dcs, deets, errs, diff --git a/src/internal/m365/sharepoint/library_handler.go b/src/internal/m365/sharepoint/library_handler.go index 07c997fcb..3f16c6eae 100644 --- a/src/internal/m365/sharepoint/library_handler.go +++ b/src/internal/m365/sharepoint/library_handler.go @@ -157,11 +157,25 @@ func (h libraryBackupHandler) IncludesDir(dir string) bool { var _ onedrive.RestoreHandler = &libraryRestoreHandler{} type libraryRestoreHandler struct { - ac api.Drives + ac api.Client +} + +func (h libraryRestoreHandler) PostDrive( + ctx context.Context, + siteID, driveName string, +) (models.Driveable, error) { + return h.ac.Lists().PostDrive(ctx, siteID, driveName) } func NewRestoreHandler(ac api.Client) *libraryRestoreHandler { - return &libraryRestoreHandler{ac.Drives()} + return &libraryRestoreHandler{ac} +} + +func (h libraryRestoreHandler) NewDrivePager( + resourceOwner string, + fields []string, +) api.DrivePager { + return h.ac.Drives().NewSiteDrivePager(resourceOwner, fields) } func (h libraryRestoreHandler) AugmentItemInfo( @@ -177,21 +191,21 @@ func (h libraryRestoreHandler) DeleteItem( ctx context.Context, driveID, itemID string, ) error { - return h.ac.DeleteItem(ctx, driveID, itemID) + return h.ac.Drives().DeleteItem(ctx, driveID, itemID) } func (h libraryRestoreHandler) DeleteItemPermission( ctx context.Context, driveID, itemID, permissionID string, ) error { - return h.ac.DeleteItemPermission(ctx, driveID, itemID, permissionID) + return h.ac.Drives().DeleteItemPermission(ctx, driveID, itemID, permissionID) } func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, ) (map[string]api.DriveItemIDType, error) { - m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) + m, err := h.ac.Drives().GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err } @@ -203,7 +217,7 @@ func (h libraryRestoreHandler) NewItemContentUpload( ctx context.Context, driveID, itemID string, ) (models.UploadSessionable, error) { - return h.ac.NewItemContentUpload(ctx, driveID, itemID) + return h.ac.Drives().NewItemContentUpload(ctx, driveID, itemID) } func (h libraryRestoreHandler) PostItemPermissionUpdate( @@ -211,7 +225,7 @@ func (h libraryRestoreHandler) PostItemPermissionUpdate( driveID, itemID string, body *drives.ItemItemsItemInvitePostRequestBody, ) (drives.ItemItemsItemInviteResponseable, error) { - return h.ac.PostItemPermissionUpdate(ctx, driveID, itemID, body) + return h.ac.Drives().PostItemPermissionUpdate(ctx, driveID, itemID, body) } func (h libraryRestoreHandler) PostItemLinkShareUpdate( @@ -219,7 +233,7 @@ func (h libraryRestoreHandler) PostItemLinkShareUpdate( driveID, itemID string, body *drives.ItemItemsItemCreateLinkPostRequestBody, ) (models.Permissionable, error) { - return h.ac.PostItemLinkShareUpdate(ctx, driveID, itemID, body) + return h.ac.Drives().PostItemLinkShareUpdate(ctx, driveID, itemID, body) } func (h libraryRestoreHandler) PostItemInContainer( @@ -228,21 +242,21 @@ func (h libraryRestoreHandler) PostItemInContainer( newItem models.DriveItemable, onCollision control.CollisionPolicy, ) (models.DriveItemable, error) { - return h.ac.PostItemInContainer(ctx, driveID, parentFolderID, newItem, onCollision) + return h.ac.Drives().PostItemInContainer(ctx, driveID, parentFolderID, newItem, onCollision) } func (h libraryRestoreHandler) GetFolderByName( ctx context.Context, driveID, parentFolderID, folderName string, ) (models.DriveItemable, error) { - return h.ac.GetFolderByName(ctx, driveID, parentFolderID, folderName) + return h.ac.Drives().GetFolderByName(ctx, driveID, parentFolderID, folderName) } func (h libraryRestoreHandler) GetRootFolder( ctx context.Context, driveID string, ) (models.DriveItemable, error) { - return h.ac.GetRootFolder(ctx, driveID) + return h.ac.Drives().GetRootFolder(ctx, driveID) } // --------------------------------------------------------------------------- diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index 417d6d87c..c38b82e08 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -10,6 +10,8 @@ import ( "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -33,17 +35,25 @@ func ConsumeRestoreCollections( ac api.Client, restoreCfg control.RestoreConfig, opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches() - el = errs.Local() + lrh = libraryRestoreHandler{ac} + protectedResourceID = dcs[0].FullPath().ResourceOwner() + restoreMetrics support.CollectionMetrics + caches = onedrive.NewRestoreCaches(backupDriveIDNames) + el = errs.Local() ) + err := caches.Populate(ctx, lrh, protectedResourceID) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. data.SortRestoreCollections(dcs) @@ -69,13 +79,14 @@ func ConsumeRestoreCollections( case path.LibrariesCategory: metrics, err = onedrive.RestoreCollection( ictx, - libraryRestoreHandler{ac.Drives()}, + lrh, restoreCfg, backupVersion, dc, caches, deets, opts.RestorePermissions, + control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), errs, ctr) diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index ae2c8d534..912b46743 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -46,6 +46,17 @@ type ( ) (*details.Details, error) Wait() *data.CollectionStats + + CacheItemInfoer + } + + CacheItemInfoer interface { + // CacheItemInfo is used by the consumer to cache metadata that is + // sourced from per-item info, but may be valuable to the restore at + // large. + // Ex: pairing drive ids with drive names as they appeared at the time + // of backup. + CacheItemInfo(v details.ItemInfo) } RepoMaintenancer interface { diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index e77b1104b..0f853a853 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -219,7 +219,13 @@ func (op *RestoreOperation) do( observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) - paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors) + paths, err := formatDetailsForRestoration( + ctx, + bup.Version, + op.Selectors, + deets, + op.rc, + op.Errors) if err != nil { return nil, clues.Wrap(err, "formatting paths from details") } @@ -359,6 +365,7 @@ func formatDetailsForRestoration( backupVersion int, sel selectors.Selector, deets *details.Details, + cii inject.CacheItemInfoer, errs *fault.Bus, ) ([]path.RestorePaths, error) { fds, err := sel.Reduce(ctx, deets, errs) @@ -366,6 +373,11 @@ func formatDetailsForRestoration( return nil, err } + // allow restore controllers to iterate over item metadata + for _, ent := range fds.Entries { + cii.CacheItemInfo(ent.ItemInfo) + } + paths, err := pathtransformer.GetPaths(ctx, backupVersion, fds.Items(), errs) if err != nil { return nil, clues.Wrap(err, "getting restore paths") diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 5fc5f7be8..2b4129d9f 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -52,8 +52,9 @@ type RestoreConfig struct { // Defaults to "Corso_Restore_" Location string - // Drive specifies the drive into which the data will be restored. - // If empty, data is restored to the same drive that was backed up. + // Drive specifies the name of the drive into which the data will be + // restored. If empty, data is restored to the same drive that was backed + // up. // Defaults to empty. Drive string } @@ -65,6 +66,10 @@ func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { } } +func DefaultRestoreContainerName(timeFormat dttm.TimeFormat) string { + return defaultRestoreLocation + dttm.FormatNow(timeFormat) +} + // EnsureRestoreConfigDefaults sets all non-supported values in the config // struct to the default value. func EnsureRestoreConfigDefaults( diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go new file mode 100644 index 000000000..fb6abaa48 --- /dev/null +++ b/src/pkg/services/m365/api/lists.go @@ -0,0 +1,64 @@ +package api + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Lists() Lists { + return Lists{c} +} + +// Lists is an interface-compliant provider of the client. +type Lists struct { + Client +} + +// PostDrive creates a new list of type drive. Specifically used to create +// documentLibraries for SharePoint Sites. +func (c Lists) PostDrive( + ctx context.Context, + siteID, driveName string, +) (models.Driveable, error) { + list := models.NewList() + list.SetDisplayName(&driveName) + list.SetDescription(ptr.To("corso auto-generated restore destination")) + + li := models.NewListInfo() + li.SetTemplate(ptr.To("documentLibrary")) + list.SetList(li) + + // creating a list of type documentLibrary will result in the creation + // of a new drive owned by the given site. + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists() + + newList, err := builder.Post(ctx, list, nil) + if graph.IsErrItemAlreadyExistsConflict(err) { + return nil, clues.Stack(graph.ErrItemAlreadyExistsConflict, err).WithClues(ctx) + } + + if err != nil { + return nil, graph.Wrap(ctx, err, "creating documentLibrary list") + } + + // drive information is not returned by the list creation. + drive, err := builder. + ByListId(ptr.Val(newList.GetId())). + Drive(). + Get(ctx, nil) + + return drive, graph.Wrap(ctx, err, "fetching created documentLibrary").OrNil() +} diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go new file mode 100644 index 000000000..5864427f2 --- /dev/null +++ b/src/pkg/services/m365/api/lists_test.go @@ -0,0 +1,57 @@ +package api_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/control/testdata" +) + +type ListsAPIIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func (suite *ListsAPIIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func TestListsAPIIntgSuite(t *testing.T) { + suite.Run(t, &ListsAPIIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *ListsAPIIntgSuite) TestLists_PostDrive() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + driveName = testdata.DefaultRestoreConfig("list_api_post_drive").Location + siteID = suite.its.siteID + ) + + // first post, should have no errors + list, err := acl.PostDrive(ctx, siteID, driveName) + require.NoError(t, err, clues.ToCore(err)) + // the site name cannot be set when posting, only its DisplayName. + // so we double check here that we're still getting the name we expect. + assert.Equal(t, driveName, ptr.Val(list.GetName())) + + // second post, same name, should error on name conflict] + _, err = acl.PostDrive(ctx, siteID, driveName) + require.ErrorIs(t, err, graph.ErrItemAlreadyExistsConflict, clues.ToCore(err)) +} diff --git a/website/docs/support/known-issues.md b/website/docs/support/known-issues.md index 754bddfb6..e6bc12809 100644 --- a/website/docs/support/known-issues.md +++ b/website/docs/support/known-issues.md @@ -16,8 +16,6 @@ Below is a list of known Corso issues and limitations: from M365 while a backup creation is running. The next backup creation will correct any missing data. -* SharePoint document library data can't be restored after the library has been deleted. - * Sharing information of items in OneDrive/SharePoint using sharing links aren't backed up and restored. * Permissions/Access given to a site group can't be restored. From d555501093ad7b2a8da889816d3a29e39cbc815d Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Jul 2023 19:13:09 -0600 Subject: [PATCH 04/10] add integration tests for missing drives (#3847) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issue(s) * #3652 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .github/actions/purge-m365-data/action.yml | 15 +- .github/workflows/ci_test_cleanup.yml | 1 + src/cmd/purge/scripts/onedrivePurge.ps1 | 95 ++++++- src/internal/m365/graph/errors.go | 8 +- src/internal/operations/restore_test.go | 197 -------------- .../operations/test/sharepoint_test.go | 244 +++++++++++++++++- 6 files changed, 352 insertions(+), 208 deletions(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index d7681e55b..725fc711f 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -19,7 +19,9 @@ inputs: site: description: Sharepoint site where data is to be purged. libraries: - description: List of library names within site where data is to be purged. + description: List of library names within the site where data is to be purged. + library-prefix: + description: List of library names within the site where the library will get deleted entirely. folder-prefix: description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead. older-than: @@ -76,7 +78,10 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 \ + -User ${{ inputs.user }} \ + -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") \ + -PurgeBeforeTimestamp ${{ inputs.older-than }} ################################################################################################################ # Sharepoint @@ -90,4 +95,8 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -Site ${{ inputs.site }} \ + -LibraryNameList "${{ inputs.libraries }}".split(",") \ + -FolderPrefixPurgeList ${{ inputs.folder-prefix }} \ + -LibraryPrefixDeleteList ${{ inputs.library-prefix }} \ + -PurgeBeforeTimestamp ${{ inputs.older-than }} diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 65e678e4b..9f51d3e14 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -62,6 +62,7 @@ jobs: site: ${{ vars[matrix.site] }} folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} azure-client-secret: ${{ secrets.CLIENT_SECRET }} diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index 7372245aa..b4717d35e 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -19,14 +19,17 @@ Param ( [datetime]$PurgeBeforeTimestamp, [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")] - [String[]]$FolderPrefixPurgeList + [String[]]$FolderPrefixPurgeList, + + [Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")] + [String[]]$LibraryPrefixDeleteList ) Set-StrictMode -Version 2.0 # Attempt to set network timeout to 10min [System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000 -function Get-TimestampFromName { +function Get-TimestampFromFolderName { param ( [Parameter(Mandatory = $True, HelpMessage = "Folder ")] [Microsoft.SharePoint.Client.Folder]$folder @@ -54,6 +57,36 @@ function Get-TimestampFromName { return $timestamp } + +function Get-TimestampFromListName { + param ( + [Parameter(Mandatory = $True, HelpMessage = "List ")] + [Microsoft.SharePoint.Client.List]$list + ) + + $name = $list.Title + + #fallback on list create time + [datetime]$timestamp = $list.LastItemUserModifiedDate + + try { + # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601 + if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") { + $timestamp = [System.Convert]::ToDatetime($Matches.0) + } + + # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS + if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") { + $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal") + } + } + catch {} + + Write-Verbose "List: $name, create timestamp: $timestamp" + + return $timestamp +} + function Purge-Library { [CmdletBinding(SupportsShouldProcess)] Param ( @@ -77,7 +110,7 @@ function Purge-Library { foreach ($f in $folders) { $folderName = $f.Name - $createTime = Get-TimestampFromName -Folder $f + $createTime = Get-TimestampFromFolderName -Folder $f if ($PurgeBeforeTimestamp -gt $createTime) { foreach ($p in $FolderPrefixPurgeList) { @@ -97,7 +130,7 @@ function Purge-Library { if ($f.ServerRelativeUrl -imatch "$SiteSuffix/{0,1}(.+?)/{0,1}$folderName$") { $siteRelativeParentPath = $Matches.1 } - + if ($PSCmdlet.ShouldProcess("Name: " + $f.Name + " Parent: " + $siteRelativeParentPath, "Remove folder")) { Write-Host "Deleting folder: "$f.Name" with parent: $siteRelativeParentPath" try { @@ -110,6 +143,54 @@ function Purge-Library { } } +function Delete-LibraryByPrefix { + [CmdletBinding(SupportsShouldProcess)] + Param ( + [Parameter(Mandatory = $True, HelpMessage = "Document library root")] + [String]$LibraryNamePrefix, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $True, HelpMessage = "Site suffix")] + [String[]]$SiteSuffix + ) + + Write-Host "`nDeleting library: $LibraryNamePrefix" + + $listsToDelete = @() + $lists = Get-PnPList + + foreach ($l in $lists) { + $listName = $l.Title + $createTime = Get-TimestampFromListName -List $l + + if ($PurgeBeforeTimestamp -gt $createTime) { + foreach ($p in $FolderPrefixPurgeList) { + if ($listName -like "$p*") { + $listsToDelete += $l + } + } + } + } + + Write-Host "Found"$listsToDelete.count"lists to delete" + + foreach ($l in $listsToDelete) { + $listName = $l.Title + + if ($PSCmdlet.ShouldProcess("Name: " + $l.Title + "Remove folder")) { + Write-Host "Deleting list: "$l.Title + try { + Remove-PnPList -Identity $l.Id -Force + } + catch [ System.Management.Automation.ItemNotFoundException ] { + Write-Host "List: "$f.Name" is already deleted. Skipping..." + } + } + } +} + ######## MAIN ######### # Setup SharePointPnP @@ -176,4 +257,8 @@ $FolderPrefixPurgeList = $FolderPrefixPurgeList | ForEach-Object { @($_.Split(', foreach ($library in $LibraryNameList) { Purge-Library -LibraryName $library -PurgeBeforeTimestamp $PurgeBeforeTimestamp -FolderPrefixPurgeList $FolderPrefixPurgeList -SiteSuffix $siteSuffix -} \ No newline at end of file +} + +foreach ($libraryPfx in $LibraryPrefixDeleteList) { + Delete-LibraryByPrefix -LibraryNamePrefix $libraryPfx -PurgeBeforeTimestamp $PurgeBeforeTimestamp -SiteSuffix $siteSuffix +} diff --git a/src/internal/m365/graph/errors.go b/src/internal/m365/graph/errors.go index f0df6b4ec..dad2674a4 100644 --- a/src/internal/m365/graph/errors.go +++ b/src/internal/m365/graph/errors.go @@ -271,7 +271,9 @@ func Wrap(ctx context.Context, e error, msg string) *clues.Err { e = clues.Stack(e, clues.New(mainMsg)) } - return setLabels(clues.Wrap(e, msg).WithClues(ctx).With(data...), innerMsg) + ce := clues.Wrap(e, msg).WithClues(ctx).With(data...).WithTrace(1) + + return setLabels(ce, innerMsg) } // Stack is a helper function that extracts ODataError metadata from @@ -292,7 +294,9 @@ func Stack(ctx context.Context, e error) *clues.Err { e = clues.Stack(e, clues.New(mainMsg)) } - return setLabels(clues.Stack(e).WithClues(ctx).With(data...), innerMsg) + ce := clues.Stack(e).WithClues(ctx).With(data...).WithTrace(1) + + return setLabels(ce, innerMsg) } // stackReq is a helper function that extracts ODataError metadata from diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index f02ee1731..5a314aaf4 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -10,8 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" - inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -21,7 +19,6 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" - "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" @@ -32,7 +29,6 @@ import ( "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/services/m365/api" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" "github.com/alcionai/corso/src/pkg/store" ) @@ -143,13 +139,6 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { // integration // --------------------------------------------------------------------------- -type bupResults struct { - selectorResourceOwners []string - backupID model.StableID - items int - ctrl *m365.Controller -} - type RestoreOpIntegrationSuite struct { tester.Suite @@ -267,192 +256,6 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { } } -func setupExchangeBackup( - t *testing.T, - kw *kopia.Wrapper, - sw *store.Wrapper, - acct account.Account, - owner string, -) bupResults { - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - users = []string{owner} - esel = selectors.NewExchangeBackup(users) - ) - - esel.DiscreteOwner = owner - esel.Include( - esel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()), - esel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), - esel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) - - ctrl, sel := ControllerWithSelector(t, ctx, acct, resource.Users, esel.Selector, nil, nil) - - bo, err := NewBackupOperation( - ctx, - control.Defaults(), - kw, - sw, - ctrl, - acct, - sel, - inMock.NewProvider(owner, owner), - evmock.NewBus()) - require.NoError(t, err, clues.ToCore(err)) - - err = bo.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results.BackupID) - - return bupResults{ - selectorResourceOwners: users, - backupID: bo.Results.BackupID, - // Discount metadata collection files (1 delta and one prev path for each category). - // These meta files are used to aid restore, but are not themselves - // restored (ie: counted as writes). - items: bo.Results.ItemsWritten - 6, - ctrl: ctrl, - } -} - -func setupSharePointBackup( - t *testing.T, - kw *kopia.Wrapper, - sw *store.Wrapper, - acct account.Account, - owner string, -) bupResults { - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - sites = []string{owner} - ssel = selectors.NewSharePointBackup(sites) - ) - - // assume a folder name "test" exists in the drive. - // this is brittle, and requires us to backfill anytime - // the site under test changes, but also prevents explosive - // growth from re-backup/restore of restored files. - ssel.Include(ssel.LibraryFolders([]string{"test"}, selectors.PrefixMatch())) - ssel.DiscreteOwner = owner - - ctrl, sel := ControllerWithSelector(t, ctx, acct, resource.Sites, ssel.Selector, nil, nil) - - bo, err := NewBackupOperation( - ctx, - control.Defaults(), - kw, - sw, - ctrl, - acct, - sel, - inMock.NewProvider(owner, owner), - evmock.NewBus()) - require.NoError(t, err, clues.ToCore(err)) - - err = bo.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results.BackupID) - - return bupResults{ - selectorResourceOwners: sites, - backupID: bo.Results.BackupID, - // Discount metadata files (2: 1 delta, 1 prev path) - // assume only one folder, and therefore 1 dirmeta per drive - // (2 drives: documents and more documents) - // assume only one file in each folder, and therefore 1 meta per drive - // (2 drives: documents and more documents) - // Meta files are used to aid restore, but are not themselves - // restored (ie: counted as writes). - items: bo.Results.ItemsWritten - 6, - ctrl: ctrl, - } -} - -func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { - tables := []struct { - name string - owner string - restoreCfg control.RestoreConfig - getSelector func(t *testing.T, owners []string) selectors.Selector - setup func(t *testing.T, kw *kopia.Wrapper, sw *store.Wrapper, acct account.Account, owner string) bupResults - }{ - { - name: "Exchange_Restore", - owner: tconfig.M365UserID(suite.T()), - restoreCfg: testdata.DefaultRestoreConfig(""), - getSelector: func(t *testing.T, owners []string) selectors.Selector { - rsel := selectors.NewExchangeRestore(owners) - rsel.Include(rsel.AllData()) - - return rsel.Selector - }, - setup: setupExchangeBackup, - }, - { - name: "SharePoint_Restore", - owner: tconfig.M365SiteID(suite.T()), - restoreCfg: control.DefaultRestoreConfig(dttm.SafeForTesting), - getSelector: func(t *testing.T, owners []string) selectors.Selector { - rsel := selectors.NewSharePointRestore(owners) - rsel.Include(rsel.Library(tconfig.LibraryDocuments), rsel.Library(tconfig.LibraryMoreDocuments)) - - return rsel.Selector - }, - setup: setupSharePointBackup, - }, - } - - for _, test := range tables { - suite.Run(test.name, func() { - var ( - t = suite.T() - mb = evmock.NewBus() - bup = test.setup(t, suite.kw, suite.sw, suite.acct, test.owner) - ) - - ctx, flush := tester.NewContext(t) - defer flush() - - require.NotZero(t, bup.items) - require.NotEmpty(t, bup.backupID) - - ro, err := NewRestoreOperation( - ctx, - control.Options{FailureHandling: control.FailFast}, - suite.kw, - suite.sw, - bup.ctrl, - tconfig.NewM365Account(t), - bup.backupID, - test.getSelector(t, bup.selectorResourceOwners), - test.restoreCfg, - mb, - count.New()) - require.NoError(t, err, clues.ToCore(err)) - - ds, err := ro.Run(ctx) - - require.NoError(t, err, "restoreOp.Run() %+v", clues.ToCore(err)) - require.NotEmpty(t, ro.Results, "restoreOp results") - require.NotNil(t, ds, "restored details") - assert.Equal(t, ro.Status, Completed, "restoreOp status") - assert.Equal(t, ro.Results.ItemsWritten, len(ds.Items()), "item write count matches len details") - assert.Less(t, 0, ro.Results.ItemsRead, "restore items read") - assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read") - assert.Equal(t, 1, ro.Results.ResourceOwners, "resource Owners") - assert.NoError(t, ro.Errors.Failure(), "non-recoverable error", clues.ToCore(ro.Errors.Failure())) - assert.Empty(t, ro.Errors.Recovered(), "recoverable errors") - assert.Equal(t, bup.items, ro.Results.ItemsWritten, "backup and restore wrote the same num of items") - assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") - assert.Equal(t, 1, mb.TimesCalled[events.RestoreEnd], "restore-end events") - }) - } -} - func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { t := suite.T() diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index 1b5a52dc2..ad2e5d79a 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -5,6 +5,9 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -19,6 +22,8 @@ import ( "github.com/alcionai/corso/src/internal/version" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" @@ -196,7 +201,7 @@ func (suite *SharePointRestoreIntgSuite) SetupSuite() { } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { - sel := selectors.NewSharePointBackup([]string{suite.its.userID}) + sel := selectors.NewSharePointBackup([]string{suite.its.siteID}) sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Filter(sel.Library("documents")) sel.DiscreteOwner = suite.its.siteID @@ -209,3 +214,240 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedO suite.its.siteDriveID, suite.its.siteDriveRootFolderID) } + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() { + t := suite.T() + + // despite the client having a method for drive.Patch and drive.Delete, both only return + // the error code and message `invalidRequest`. + t.Skip("graph api doesn't allow patch or delete on drives, so we cannot run any conditions") + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := ctrlTD.DefaultRestoreConfig("restore_deleted_drives") + rc.OnCollision = control.Copy + + // create a new drive + md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.siteID, rc.Location) + require.NoError(t, err, clues.ToCore(err)) + + driveID := ptr.Val(md.GetId()) + + // get the root folder + mdi, err := suite.its.ac.Drives().GetRootFolder(ctx, driveID) + require.NoError(t, err, clues.ToCore(err)) + + rootFolderID := ptr.Val(mdi.GetId()) + + // add an item to it + itemName := uuid.NewString() + + item := models.NewDriveItem() + item.SetName(ptr.To(itemName + ".txt")) + + file := models.NewFile() + item.SetFile(file) + + _, err = suite.its.ac.Drives().PostItemInContainer( + ctx, + driveID, + rootFolderID, + item, + control.Copy) + require.NoError(t, err, clues.ToCore(err)) + + // run a backup + var ( + mb = evmock.NewBus() + opts = control.Defaults() + graphClient = suite.its.ac.Stable.Client() + ) + + bsel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + bsel.Include(selTD.SharePointBackupFolderScope(bsel)) + bsel.Filter(bsel.Library(rc.Location)) + bsel.DiscreteOwner = suite.its.siteID + + bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + // test cases: + + // first test, we take the current drive and rename it. + // the restore should find the drive by id and restore items + // into it like normal. Due to collision handling, this should + // create a copy of the current item. + suite.Run("renamed drive", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + patchBody := models.NewDrive() + patchBody.SetName(ptr.To("some other name")) + + md, err = graphClient. + Drives(). + ByDriveId(driveID). + Patch(ctx, patchBody, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 2) + + for _, item := range items { + assert.Contains(t, ptr.Val(item.GetName()), itemName) + } + }) + + // second test, we delete the drive altogether. the restore should find + // no existing drives, but it should have the old drive's name and attempt + // to recreate that drive by name. + suite.Run("deleted drive", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + err = graphClient. + Drives(). + ByDriveId(driveID). + Delete(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + pgr := suite.its.ac. + Drives(). + NewSiteDrivePager(suite.its.siteID, []string{"id", "name"}) + + drives, err := api.GetAllDrives(ctx, pgr, false, -1) + require.NoError(t, err, clues.ToCore(err)) + + var created models.Driveable + + for _, drive := range drives { + if ptr.Val(drive.GetName()) == ptr.Val(created.GetName()) && + ptr.Val(drive.GetId()) != driveID { + created = drive + break + } + } + + require.NotNil(t, created, "found the restored drive by name") + md = created + driveID = ptr.Val(md.GetId()) + + mdi, err := suite.its.ac.Drives().GetRootFolder(ctx, driveID) + require.NoError(t, err, clues.ToCore(err)) + + rootFolderID = ptr.Val(mdi.GetId()) + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 1) + + assert.Equal(t, ptr.Val(items[0].GetName()), itemName+".txt") + }) + + // final test, run a follow-up restore. This should match the + // drive we created in the prior test by name, but not by ID. + suite.Run("different drive - same name", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 2) + + for _, item := range items { + assert.Contains(t, ptr.Val(item.GetName()), itemName) + } + }) +} From 5ac0c135bd3dfa11a621374c91287da7fd9c6b2f Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Jul 2023 20:22:03 -0600 Subject: [PATCH 05/10] single-line cleanup scripts in gh (#3871) - [x] :bug: Bugfix #### Test Plan - [x] :green_heart: E2E --- .github/actions/purge-m365-data/action.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index 725fc711f..ec26b987d 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -78,10 +78,7 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 \ - -User ${{ inputs.user }} \ - -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") \ - -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} ################################################################################################################ # Sharepoint @@ -95,8 +92,4 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} \ - -LibraryNameList "${{ inputs.libraries }}".split(",") \ - -FolderPrefixPurgeList ${{ inputs.folder-prefix }} \ - -LibraryPrefixDeleteList ${{ inputs.library-prefix }} \ - -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} From 1f7941d19bdd4a1463f086bab869975e55143f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 06:04:51 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20sass=20from?= =?UTF-8?q?=201.63.6=20to=201.64.0=20in=20/website=20(#3874)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sass](https://github.com/sass/dart-sass) from 1.63.6 to 1.64.0.
Release notes

Sourced from sass's releases.

Dart Sass 1.64.0

To install Sass 1.64.0, download one of the packages below and add it to your PATH, or see the Sass website for full installation instructions.

Changes

  • Comments that appear before or between @use and @forward rules are now emitted in source order as much as possible, instead of always being emitted after the CSS of all module dependencies.

  • Fix a bug where an interpolation in a custom property name crashed if the file was loaded by a @use nested in an @import.

JavaScript API

  • Add a new SassCalculation type that represents the calculation objects added in Dart Sass 1.40.0.

  • Add Value.assertCalculation(), which returns the value if it's a SassCalculation and throws an error otherwise.

  • Produce a better error message when an environment that supports some Node.js APIs loads the browser entrypoint but attempts to access the filesystem.

Embedded Sass

  • Fix a bug where nested relative @imports failed to load when using the deprecated functions render or renderSync and those relative imports were loaded multiple times across different files.

See the full changelog for changes in earlier releases.

Changelog

Sourced from sass's changelog.

1.64.0

  • Comments that appear before or between @use and @forward rules are now emitted in source order as much as possible, instead of always being emitted after the CSS of all module dependencies.

  • Fix a bug where an interpolation in a custom property name crashed if the file was loaded by a @use nested in an @import.

JavaScript API

  • Add a new SassCalculation type that represents the calculation objects added in Dart Sass 1.40.0.

  • Add Value.assertCalculation(), which returns the value if it's a SassCalculation and throws an error otherwise.

  • Produce a better error message when an environment that supports some Node.js APIs loads the browser entrypoint but attempts to access the filesystem.

Embedded Sass

  • Fix a bug where nested relative @imports failed to load when using the deprecated functions render or renderSync and those relative imports were loaded multiple times across different files.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sass&package-manager=npm_and_yarn&previous-version=1.63.6&new-version=1.64.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 14 +++++++------- website/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 21df1ab55..a4561faa5 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -24,7 +24,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.63.6", + "sass": "^1.64.0", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" @@ -12571,9 +12571,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.63.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", - "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.0.tgz", + "integrity": "sha512-m7YtAGmQta9uANIUJwXesAJMSncqH+3INc8kdVXs6eV6GUC8Qu2IYKQSN8PRLgiQfpca697G94klm2leYMxSHw==", "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -23802,9 +23802,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass": { - "version": "1.63.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", - "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.0.tgz", + "integrity": "sha512-m7YtAGmQta9uANIUJwXesAJMSncqH+3INc8kdVXs6eV6GUC8Qu2IYKQSN8PRLgiQfpca697G94klm2leYMxSHw==", "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", diff --git a/website/package.json b/website/package.json index 2c7272ce9..55573ce46 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.63.6", + "sass": "^1.64.0", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" From 876f0d8bd90abe4a28b9d5aab8e29ae41019d436 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 06:10:03 +0000 Subject: [PATCH 07/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20docusaurus-pl?= =?UTF-8?q?ugin-sass=20from=200.2.4=20to=200.2.5=20in=20/website=20(#3873)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docusaurus-plugin-sass](https://github.com/rlamana/docusaurus-plugin-sass) from 0.2.4 to 0.2.5.
Release notes

Sourced from docusaurus-plugin-sass's releases.

v0.2.5

What's Changed

New Contributors

Full Changelog: https://github.com/rlamana/docusaurus-plugin-sass/compare/v0.2.4...v0.2.5-2

Commits
  • 70e42fc Merge pull request #21 from roydukkey/issue/20
  • 7a5720c Merge pull request #33 from noomorph/patch-1
  • d9f4fad build: support docusarus@3.0.0-alpha.0
  • 2fe9818 Merge branch 'rlamana:master' into issue/20
  • bb70d7d Add missing newline
  • c482d9a Add TypeScript support for Sass/SCSS modules
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docusaurus-plugin-sass&package-manager=npm_and_yarn&previous-version=0.2.4&new-version=0.2.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 16 ++++++++-------- website/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index a4561faa5..bbf48e4b8 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -16,7 +16,7 @@ "animate.css": "^4.1.1", "clsx": "^2.0.0", "docusaurus-plugin-image-zoom": "^1.0.1", - "docusaurus-plugin-sass": "^0.2.4", + "docusaurus-plugin-sass": "^0.2.5", "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", @@ -6541,14 +6541,14 @@ } }, "node_modules/docusaurus-plugin-sass": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.4.tgz", - "integrity": "sha512-r9bLXW6X2z64bzQUQZB1SxmNlGvSO9swTFALgiMjr/1O4FRDti6BseU4Sw2mlZkYvVQTq8cJMJIP6w7z/5We8Q==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", + "integrity": "sha512-Z+D0fLFUKcFpM+bqSUmqKIU+vO+YF1xoEQh5hoFreg2eMf722+siwXDD+sqtwU8E4MvVpuvsQfaHwODNlxJAEg==", "dependencies": { "sass-loader": "^10.1.1" }, "peerDependencies": { - "@docusaurus/core": "^2.0.0-beta", + "@docusaurus/core": "^2.0.0-beta || ^3.0.0-alpha", "sass": "^1.30.0" } }, @@ -19867,9 +19867,9 @@ } }, "docusaurus-plugin-sass": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.4.tgz", - "integrity": "sha512-r9bLXW6X2z64bzQUQZB1SxmNlGvSO9swTFALgiMjr/1O4FRDti6BseU4Sw2mlZkYvVQTq8cJMJIP6w7z/5We8Q==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", + "integrity": "sha512-Z+D0fLFUKcFpM+bqSUmqKIU+vO+YF1xoEQh5hoFreg2eMf722+siwXDD+sqtwU8E4MvVpuvsQfaHwODNlxJAEg==", "requires": { "sass-loader": "^10.1.1" } diff --git a/website/package.json b/website/package.json index 55573ce46..3e1388b77 100644 --- a/website/package.json +++ b/website/package.json @@ -22,7 +22,7 @@ "animate.css": "^4.1.1", "clsx": "^2.0.0", "docusaurus-plugin-image-zoom": "^1.0.1", - "docusaurus-plugin-sass": "^0.2.4", + "docusaurus-plugin-sass": "^0.2.5", "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", From de742bbd2f40154871466409c12eab6c7f040e1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:13:15 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.304=20to=201.44.305=20in=20/src=20(#?= =?UTF-8?q?3867)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.304 to 1.44.305.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.305 (2023-07-20)

Service Client Updates

  • service/connectcases: Updates service API and documentation
  • service/models.lex.v2: Updates service API and documentation
  • service/route53resolver: Updates service API, documentation, and paginators
  • service/s3: Updates service examples
    • Improve performance of S3 clients by simplifying and optimizing endpoint resolution.
  • service/sagemaker: Updates service API, documentation, and paginators
    • Cross account support for SageMaker Feature Store
  • service/sagemaker-featurestore-runtime: Updates service API and documentation
  • service/securitylake: Updates service API and documentation
  • service/transcribe: Updates service API and documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.304&new-version=1.44.305)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 6ebbb2adb..5cb7b980e 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.304 + github.com/aws/aws-sdk-go v1.44.305 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index a66eb73f4..fe18cc952 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.304 h1:crcJBVeewWcVAXDQChzJWZYGFq9i1TYLycAtQ6Xpi4c= -github.com/aws/aws-sdk-go v1.44.304/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.305 h1:fU/5lY3WyBjGU9fkmQYd8o4fZu+2RaOv/i+sPaJVvFg= +github.com/aws/aws-sdk-go v1.44.305/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 5c80d7645525cede346fcdc87c5b62c832876394 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 21 Jul 2023 10:56:26 -0600 Subject: [PATCH 09/10] give library prefix delete list a default val (#3872) #### Type of change - [x] :bug: Bugfix --- .github/actions/purge-m365-data/action.yml | 9 +++------ .github/workflows/ci_test_cleanup.yml | 14 ++++++-------- src/cmd/purge/scripts/onedrivePurge.ps1 | 2 +- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index ec26b987d..dd67f2326 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -53,8 +53,7 @@ runs: AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} - run: | - ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} - name: Reset retention for all mailboxes to 0 if: ${{ inputs.user == '' }} @@ -63,8 +62,7 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./exchangeRetention.ps1 + run: ./exchangeRetention.ps1 ################################################################################################################ # OneDrive @@ -91,5 +89,4 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 9f51d3e14..11ba94097 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -12,7 +12,7 @@ jobs: continue-on-error: true strategy: matrix: - user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ] + user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, '' ] steps: - uses: actions/checkout@v3 @@ -22,8 +22,7 @@ jobs: # At 20 minutes ago, we should be safe from conflicts. # The additional 10 minutes is just to be good citizens. - name: Set purge boundary - run: | - echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge CI-Produced Folders for Users uses: ./.github/actions/purge-m365-data @@ -43,7 +42,7 @@ jobs: continue-on-error: true strategy: matrix: - site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] + site: [ CORSO_M365_TEST_SITE_URL ] steps: - uses: actions/checkout@v3 @@ -53,15 +52,14 @@ jobs: # At 20 minutes ago, we should be safe from conflicts. # The additional 10 minutes is just to be good citizens. - name: Set purge boundary - run: | - echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge CI-Produced Folders for Sites uses: ./.github/actions/purge-m365-data with: site: ${{ vars[matrix.site] }} - folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} - libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} + libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index b4717d35e..e8f258b95 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -22,7 +22,7 @@ Param ( [String[]]$FolderPrefixPurgeList, [Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")] - [String[]]$LibraryPrefixDeleteList + [String[]]$LibraryPrefixDeleteList = @() ) Set-StrictMode -Version 2.0 From 47e5707ae4188f3fb3c82deedbacdf8d3aa1eaef Mon Sep 17 00:00:00 2001 From: ryanfkeepers Date: Fri, 21 Jul 2023 12:51:28 -0600 Subject: [PATCH 10/10] update all graph packages --- src/go.mod | 13 ++++++------- src/go.sum | 26 ++++++++++++-------------- src/internal/m365/graph/errors.go | 2 +- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/go.mod b/src/go.mod index 5cb7b980e..b589ca542 100644 --- a/src/go.mod +++ b/src/go.mod @@ -5,7 +5,7 @@ go 1.20 replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 require ( - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go v1.44.305 @@ -14,12 +14,12 @@ require ( github.com/google/uuid v1.3.0 github.com/h2non/gock v1.2.0 github.com/kopia/kopia v0.12.2-0.20230327171220-747baeebdab1 - github.com/microsoft/kiota-abstractions-go v1.0.0 + github.com/microsoft/kiota-abstractions-go v1.1.0 github.com/microsoft/kiota-authentication-azure-go v1.0.0 github.com/microsoft/kiota-http-go v1.0.0 github.com/microsoft/kiota-serialization-form-go v1.0.0 - github.com/microsoft/kiota-serialization-json-go v1.0.2 - github.com/microsoftgraph/msgraph-sdk-go v1.4.0 + github.com/microsoft/kiota-serialization-json-go v1.0.4 + github.com/microsoftgraph/msgraph-sdk-go v1.12.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/puzpuzpuz/xsync/v2 v2.4.1 @@ -42,7 +42,6 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect @@ -62,9 +61,9 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/src/go.sum b/src/go.sum index fe18cc952..718084162 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= @@ -102,7 +102,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= @@ -272,20 +271,20 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/kiota-abstractions-go v1.0.0 h1:teQS3yOmcTyps+O48AD17LI8TR1B3wCEwGFcwC6K75c= -github.com/microsoft/kiota-abstractions-go v1.0.0/go.mod h1:2yaRQnx2KU7UaenYSApiTT4pf7fFkPV0B71Rm2uYynQ= +github.com/microsoft/kiota-abstractions-go v1.1.0 h1:X1aKlsYCRs/0RSChr/fbq4j/+kxRzbSY5GeWhtHQNYI= +github.com/microsoft/kiota-abstractions-go v1.1.0/go.mod h1:RkxyZ5x87Njik7iVeQY9M2wtrrL1MJZcXiI/BxD/82g= github.com/microsoft/kiota-authentication-azure-go v1.0.0 h1:29FNZZ/4nnCOwFcGWlB/sxPvWz487HA2bXH8jR5k2Rk= github.com/microsoft/kiota-authentication-azure-go v1.0.0/go.mod h1:rnx3PRlkGdXDcA/0lZQTbBwyYGmc+3POt7HpE/e4jGw= github.com/microsoft/kiota-http-go v1.0.0 h1:F1hd6gMlLeEgH2CkRB7z13ow7LxMKMWEmms/t0VfS+k= github.com/microsoft/kiota-http-go v1.0.0/go.mod h1:eujxJliqodotsYepIc6ihhK+vXMMt5Q8YiSNL7+7M7U= github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjblsyVgifS11b3WCx+eFEsAI= github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA= -github.com/microsoft/kiota-serialization-json-go v1.0.2 h1:RXan8v7yWBD88XxVZ2W38BBcqu2UqWtgS54nCbOS5ow= -github.com/microsoft/kiota-serialization-json-go v1.0.2/go.mod h1:AUItT9exyxmjZQE8IeFD9ygP77q9GKVb+AQE2V5Ikho= +github.com/microsoft/kiota-serialization-json-go v1.0.4 h1:5TaISWwd2Me8clrK7SqNATo0tv9seOq59y4I5953egQ= +github.com/microsoft/kiota-serialization-json-go v1.0.4/go.mod h1:rM4+FsAY+9AEpBsBzkFFis+b/LZLlNKKewuLwK9Q6Mg= github.com/microsoft/kiota-serialization-text-go v1.0.0 h1:XOaRhAXy+g8ZVpcq7x7a0jlETWnWrEum0RhmbYrTFnA= github.com/microsoft/kiota-serialization-text-go v1.0.0/go.mod h1:sM1/C6ecnQ7IquQOGUrUldaO5wj+9+v7G2W3sQ3fy6M= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0 h1:ibNwMDEZ6HikA9BVXu+TljCzCiE+yFsD6wLpJbTc1tc= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0/go.mod h1:JIDL1xENx92B60NjO2ACyqGeKvtYkdl9rirgajIgryw= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0 h1:/jZJ1KCtVlvxStKq31VsEPOQQ5Iy26R1pgvc+RYt7XI= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0/go.mod h1:ccLv84FJFtwdSzYWM/HlTes5FLzkzzBsYh9kg93/WS8= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8wCKPgpODs/D1cUZ3fkAY= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -303,7 +302,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= diff --git a/src/internal/m365/graph/errors.go b/src/internal/m365/graph/errors.go index dad2674a4..f35b91385 100644 --- a/src/internal/m365/graph/errors.go +++ b/src/internal/m365/graph/errors.go @@ -365,7 +365,7 @@ func errData(err odataerrors.ODataErrorable) (string, []any, string) { msgConcat += ptr.Val(d.GetMessage()) } - inner := mainErr.GetInnererror() + inner := mainErr.GetInnerError() if inner != nil { data = appendIf(data, "odataerror_inner_cli_req_id", inner.GetClientRequestId()) data = appendIf(data, "odataerror_inner_req_id", inner.GetRequestId())