From 34d6c18d74698b3ec8e0318e855dd498d4e93bf7 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 27 Jul 2023 13:16:11 -0600 Subject: [PATCH 01/17] clean up exchange adv restore test (#3870) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/operations/test/exchange_test.go | 351 +++++++----------- 1 file changed, 138 insertions(+), 213 deletions(-) diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index ddb16216b..33bd18a6a 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -889,6 +889,26 @@ func (suite *ExchangeRestoreIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } +type clientItemPager interface { + GetItemsInContainerByCollisionKeyer[string] + GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, + ) (map[string]struct{}, error) + GetContainerByName( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]string, error) + CreateContainer( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) +} + func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() { t := suite.T() @@ -921,25 +941,28 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio require.NoError(t, err, clues.ToCore(err)) var ( - restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") - sel = rsel.Selector - userID = sel.ID() - cIDs = map[path.CategoryType]string{ - path.ContactsCategory: "", - path.EmailCategory: "", - path.EventsCategory: "", + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") + sel = rsel.Selector + userID = sel.ID() + countItemsInRestore int + + itemIDs = map[path.CategoryType]map[string]struct{}{} + collisionKeys = map[path.CategoryType]map[string]string{} + containerIDs = map[path.CategoryType]string{} + parentContainerIDs = map[path.CategoryType]string{ + path.EmailCategory: api.MsgFolderRoot, + } + parentContainerNames = map[path.CategoryType][]string{ + path.EmailCategory: {api.MailInbox}, + path.ContactsCategory: {}, + path.EventsCategory: {}, + } + + testCategories = map[path.CategoryType]clientItemPager{ + path.ContactsCategory: suite.its.ac.Contacts(), + path.EmailCategory: suite.its.ac.Mail(), + // path.EventsCategory: suite.its.ac.Events(), } - collKeys = map[path.CategoryType]map[string]string{} - countContactsInRestore int - acCont = suite.its.ac.Contacts() - contactIDs map[string]struct{} - countEmailsInRestore int - acMail = suite.its.ac.Mail() - mailIDs map[string]struct{} - countItemsInRestore int - // countEventsInRestore int - // acEvts = suite.its.ac.Events() - // eventIDs = []string{} ) // initial restore @@ -971,61 +994,27 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio // get all files in folder, use these as the base // set of files to compare against. - // --- contacts + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - contGC, err := acCont.GetContainerByName(ctx, userID, "", restoreCfg.Location) - require.NoError(t, err, clues.ToCore(err)) + ctx, flush := tester.NewContext(t) + defer flush() - cIDs[path.ContactsCategory] = ptr.Val(contGC.GetId()) + containers := append([]string{restoreCfg.Location}, parentContainerNames[cat]...) - collKeys[path.ContactsCategory], err = acCont.GetItemsInContainerByCollisionKey( - ctx, - userID, - cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) - countContactsInRestore = len(collKeys[path.ContactsCategory]) - t.Log(countContactsInRestore, "contacts restored") + itemIDs[cat], collisionKeys[cat], containerIDs[cat] = getCollKeysAndItemIDs( + t, + ctx, + ac, + userID, + parentContainerIDs[cat], + containers...) - contactIDs, err = acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + countItemsInRestore += len(collisionKeys[cat]) + }) + } - // --- events - - // gc, err = acEvts.GetContainerByName(ctx, userID, "", restoreCfg.Location) - // require.NoError(t, err, clues.ToCore(err)) - - // restoredContainerID[path.EventsCategory] = ptr.Val(gc.GetId()) - - // collKeys[path.EventsCategory], err = acEvts.GetItemsInContainerByCollisionKey( - // ctx, - // userID, - // cIDs[path.EventsCategory]) - // require.NoError(t, err, clues.ToCore(err)) - // countEventsInRestore = len(collKeys[path.EventsCategory]) - // t.Log(countContactsInRestore, "events restored") - - mailGC, err := acMail.GetContainerByName(ctx, userID, api.MsgFolderRoot, restoreCfg.Location) - require.NoError(t, err, clues.ToCore(err)) - - mailGC, err = acMail.GetContainerByName(ctx, userID, ptr.Val(mailGC.GetId()), api.MailInbox) - require.NoError(t, err, clues.ToCore(err)) - - cIDs[path.EmailCategory] = ptr.Val(mailGC.GetId()) - - // --- mail - - collKeys[path.EmailCategory], err = acMail.GetItemsInContainerByCollisionKey( - ctx, - userID, - cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - countEmailsInRestore = len(collKeys[path.EmailCategory]) - t.Log(countContactsInRestore, "emails restored") - - mailIDs, err = acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - countItemsInRestore = countContactsInRestore + countEmailsInRestore // + countEventsInRestore checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore) }) @@ -1062,43 +1051,30 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0) - // --- contacts + result := map[string]string{} - // get all files in folder, use these as the base - // set of files to compare against. - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + ctx, flush := tester.NewContext(t) + defer flush() - assert.Equal(t, contactIDs, currentContactIDs, "ids are equal") + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) - // --- events + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) - - // --- mail - - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) - - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, mailIDs, currentMailIDs, "ids are equal") + assert.Equal(t, itemIDs[cat], currentIDs, "ids are equal") + }) + } assert.Len(t, result, 0, "no new items should get added") }) @@ -1136,60 +1112,40 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio } } - assert.Len( - t, - filtEnts, - countItemsInRestore, - "every item should have been replaced") - - // --- contacts - - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) - - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, len(contactIDs), len(currentContactIDs), "count of ids are equal") - for orig := range contactIDs { - assert.NotContains(t, currentContactIDs, orig, "original item should not exist after replacement") - } - - contactIDs = currentContactIDs - - // --- events - - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) - - // --- mail - - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) + assert.Len(t, filtEnts, countItemsInRestore, "every item should have been replaced") checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0) - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) + result := map[string]string{} - assert.Equal(t, len(mailIDs), len(currentMailIDs), "count of ids are equal") - for orig := range mailIDs { - assert.NotContains(t, currentMailIDs, orig, "original item should not exist after replacement") + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) + + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(itemIDs[cat]), len(currentIDs), "count of ids are equal") + for orig := range itemIDs[cat] { + assert.NotContains(t, currentIDs, orig, "original item should not exist after replacement") + } + + itemIDs[cat] = currentIDs + }) } - mailIDs = currentMailIDs - assert.Len(t, result, 0, "all items should have been replaced") }) @@ -1226,45 +1182,35 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio } } - assert.Len( - t, - filtEnts, - countItemsInRestore, - "every item should have been copied") + assert.Len(t, filtEnts, countItemsInRestore, "every item should have been copied") checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore) - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) + result := map[string]string{} - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - assert.Equal(t, 2*len(contactIDs), len(currentContactIDs), "count of ids should be double from before") - assert.Subset(t, maps.Keys(currentContactIDs), maps.Keys(contactIDs), "original item should exist after copy") + ctx, flush := tester.NewContext(t) + defer flush() - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, 2*len(mailIDs), len(currentMailIDs), "count of ids should be double from before") - assert.Subset(t, maps.Keys(currentMailIDs), maps.Keys(mailIDs), "original item should exist after copy") + assert.Equal(t, 2*len(itemIDs[cat]), len(currentIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentIDs), maps.Keys(itemIDs[cat]), "original item should exist after copy") + }) + } // TODO: we have the option of modifying copy creations in exchange // so that the results don't collide. But we haven't made that @@ -1344,7 +1290,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- contacts cat := path.ContactsCategory - userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acCont, @@ -1354,7 +1300,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- events // cat = path.EventsCategory - // userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + // userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( // t, // ctx, // acEvts, @@ -1364,7 +1310,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- mail cat = path.EmailCategory - userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acMail, @@ -1400,7 +1346,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- contacts cat = path.ContactsCategory - secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acCont, @@ -1410,7 +1356,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- events // cat = path.EventsCategory - // secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + // secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( // t, // ctx, // acEvts, @@ -1420,7 +1366,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- mail cat = path.EmailCategory - secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acMail, @@ -1436,52 +1382,31 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte } } -type GetItemsKeysAndContainerByNameer interface { - GetItemIDsInContainer( - ctx context.Context, - userID, containerID string, - ) (map[string]struct{}, error) - GetContainerByName( - ctx context.Context, - userID, parentContainerID, containerName string, - ) (graph.Container, error) - GetItemsInContainerByCollisionKey( - ctx context.Context, - userID, containerID string, - ) (map[string]string, error) -} - func getCollKeysAndItemIDs( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument - gikacbn GetItemsKeysAndContainerByNameer, + cip clientItemPager, userID, parentContainerID string, containerNames ...string, -) (map[string]struct{}, map[string]string) { +) (map[string]struct{}, map[string]string, string) { var ( c graph.Container err error - cID string + cID = parentContainerID ) for _, cn := range containerNames { - pcid := parentContainerID - - if len(cID) != 0 { - pcid = cID - } - - c, err = gikacbn.GetContainerByName(ctx, userID, pcid, cn) + c, err = cip.GetContainerByName(ctx, userID, cID, cn) require.NoError(t, err, clues.ToCore(err)) cID = ptr.Val(c.GetId()) } - itemIDs, err := gikacbn.GetItemIDsInContainer(ctx, userID, cID) + itemIDs, err := cip.GetItemIDsInContainer(ctx, userID, cID) require.NoError(t, err, clues.ToCore(err)) - collisionKeys, err := gikacbn.GetItemsInContainerByCollisionKey(ctx, userID, cID) + collisionKeys, err := cip.GetItemsInContainerByCollisionKey(ctx, userID, cID) require.NoError(t, err, clues.ToCore(err)) - return itemIDs, collisionKeys + return itemIDs, collisionKeys, cID } From 4e2ee2484fe5341e718947dc5d0a599c92c27adb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:47:38 +0000 Subject: [PATCH 02/17] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.308=20to=201.44.309=20in=20/src=20(#?= =?UTF-8?q?3918)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.308 to 1.44.309.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.309 (2023-07-26)

Service Client Updates

  • service/cloudcontrol: Updates service documentation
  • service/entityresolution: Adds new service
  • service/glue: Updates service API and documentation
    • Release Glue Studio Snowflake Connector Node for SDK/CLI
  • service/healthlake: Updates service documentation
  • service/managedblockchain-query: Adds new service
  • service/mediaconvert: Updates service API and documentation
    • This release includes general updates to user documentation.
  • service/omics: Updates service documentation
  • service/opensearchserverless: Updates service API and documentation
  • service/polly: Updates service API
    • Amazon Polly adds 1 new voice - Lisa (nl-BE)
  • service/route53: Updates service documentation
    • Update that corrects the documents for received feedback.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.308&new-version=1.44.309)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 04a4d8c51..992c60f69 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.308 + github.com/aws/aws-sdk-go v1.44.309 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index f40e14356..0c1ea866b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.308 h1:XKu+76UHsD5LaiU2Zb1q42uWakw80Az7x39jJXXahos= -github.com/aws/aws-sdk-go v1.44.308/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.309 h1:IPJOFBzXekakxmEpDwd4RTKmmBR6LIAiXgNsM51bWbU= +github.com/aws/aws-sdk-go v1.44.309/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From d7443c2211a393f0a56892208833a049e8408021 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 13:31:12 +0530 Subject: [PATCH 03/17] CLI for exporting data from OneDrive (#3822) This adds the final piece to enable OneDrive exports. The CLI interface which consumes NewExport interface from Repository and lets the user to export and write the contents in a backup to the local filesystem. Prev: https://github.com/alcionai/corso/pull/3821 Next: https://github.com/alcionai/corso/pull/3824 --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * https://github.com/alcionai/corso/pull/3797 * https://github.com/alcionai/corso/issues/3670 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 3 + src/cli/cli.go | 4 +- src/cli/export/export.go | 39 +++++ src/cli/export/export_test.go | 175 ++++++++++++++++++++++ src/cli/export/onedrive.go | 215 ++++++++++++++++++++++++++++ src/cli/export/onedrive_test.go | 106 ++++++++++++++ src/cli/flags/export.go | 15 ++ src/cli/utils/export_config.go | 38 +++++ src/cli/utils/export_config_test.go | 54 +++++++ src/cli/utils/onedrive.go | 2 + src/cli/utils/testdata/flags.go | 4 + src/internal/observe/observe.go | 46 ++++++ src/pkg/control/restore.go | 6 +- 13 files changed, 703 insertions(+), 4 deletions(-) create mode 100644 src/cli/export/export.go create mode 100644 src/cli/export/export_test.go create mode 100644 src/cli/export/onedrive.go create mode 100644 src/cli/export/onedrive_test.go create mode 100644 src/cli/flags/export.go create mode 100644 src/cli/utils/export_config.go create mode 100644 src/cli/utils/export_config_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7123822..5170dc50f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SharePoint document libraries deleted after the last backup can now be restored. - Restore requires the protected resource to have access to the service being restored. +### Added +- Added option to export data from OneDrive backups as individual files or as a single zip file. + ## [v0.11.1] (beta) - 2023-07-20 ### Fixed diff --git a/src/cli/cli.go b/src/cli/cli.go index 5f03364a2..68f454407 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/cli/backup" "github.com/alcionai/corso/src/cli/config" + "github.com/alcionai/corso/src/cli/export" "github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/help" "github.com/alcionai/corso/src/cli/print" @@ -53,7 +54,7 @@ func preRun(cc *cobra.Command, args []string) error { } avoidTheseCommands := []string{ - "corso", "env", "help", "backup", "details", "list", "restore", "delete", "repo", "init", "connect", + "corso", "env", "help", "backup", "details", "list", "restore", "export", "delete", "repo", "init", "connect", } if len(logger.ResolvedLogFile) > 0 && !slices.Contains(avoidTheseCommands, cc.Use) { @@ -150,6 +151,7 @@ func BuildCommandTree(cmd *cobra.Command) { repo.AddCommands(cmd) backup.AddCommands(cmd) restore.AddCommands(cmd) + export.AddCommands(cmd) help.AddCommands(cmd) } diff --git a/src/cli/export/export.go b/src/cli/export/export.go new file mode 100644 index 000000000..d3ec09a8a --- /dev/null +++ b/src/cli/export/export.go @@ -0,0 +1,39 @@ +package export + +import ( + "github.com/spf13/cobra" +) + +var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ + addOneDriveCommands, +} + +// AddCommands attaches all `corso export * *` commands to the parent. +func AddCommands(cmd *cobra.Command) { + exportC := exportCmd() + cmd.AddCommand(exportC) + + for _, addExportTo := range exportCommands { + addExportTo(exportC) + } +} + +const exportCommand = "export" + +// The export category of commands. +// `corso export [] [...]` +func exportCmd() *cobra.Command { + return &cobra.Command{ + Use: exportCommand, + Short: "Export your service data", + Long: `Export the data stored in one of your M365 services.`, + RunE: handleExportCmd, + Args: cobra.NoArgs, + } +} + +// Handler for flat calls to `corso export`. +// Produces the same output as `corso export --help`. +func handleExportCmd(cmd *cobra.Command, args []string) error { + return cmd.Help() +} diff --git a/src/cli/export/export_test.go b/src/cli/export/export_test.go new file mode 100644 index 000000000..f3df68177 --- /dev/null +++ b/src/cli/export/export_test.go @@ -0,0 +1,175 @@ +package export + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/export" +) + +type ExportE2ESuite struct { + tester.Suite + called bool +} + +func TestExportE2ESuite(t *testing.T) { + suite.Run(t, &ExportE2ESuite{Suite: tester.NewE2ESuite(t, nil)}) +} + +func (suite *ExportE2ESuite) SetupSuite() { + suite.called = true +} + +type mockExportCollection struct { + path string + items []export.Item +} + +func (mec mockExportCollection) BasePath() string { return mec.path } +func (mec mockExportCollection) Items(context.Context) <-chan export.Item { + ch := make(chan export.Item) + + go func() { + defer close(ch) + + for _, item := range mec.items { + ch <- item + } + }() + + return ch +} + +func (suite *ExportE2ESuite) TestWriteExportCollection() { + type ei struct { + name string + body string + } + + type i struct { + path string + items []ei + } + + table := []struct { + name string + cols []i + }{ + { + name: "single root collection single item", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + }, + }, + }, + }, + { + name: "single root collection multiple items", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + { + name: "name2", + body: "body2", + }, + }, + }, + }, + }, + { + name: "multiple collections multiple items", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + { + name: "name2", + body: "body2", + }, + }, + }, + { + path: "folder", + items: []ei{ + { + name: "name3", + body: "body3", + }, + }, + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ecs := []export.Collection{} + for _, col := range test.cols { + items := []export.Item{} + for _, item := range col.items { + items = append(items, export.Item{ + Data: export.ItemData{ + Name: item.name, + Body: io.NopCloser((bytes.NewBufferString(item.body))), + }, + }) + } + + ecs = append(ecs, mockExportCollection{ + path: col.path, + items: items, + }) + } + + dir, err := os.MkdirTemp("", "export-test") + require.NoError(t, err) + defer os.RemoveAll(dir) + + err = writeExportCollections(ctx, dir, ecs) + require.NoError(t, err, "writing data") + + for _, col := range test.cols { + for _, item := range col.items { + f, err := os.Open(filepath.Join(dir, col.path, item.name)) + require.NoError(t, err, "opening file") + + buf := new(bytes.Buffer) + + _, err = buf.ReadFrom(f) + require.NoError(t, err, "reading file") + + assert.Equal(t, item.body, buf.String(), "file contents") + } + } + }) + } +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go new file mode 100644 index 000000000..6e715153d --- /dev/null +++ b/src/cli/export/onedrive.go @@ -0,0 +1,215 @@ +package export + +import ( + "context" + "io" + "os" + ospath "path" + + "github.com/alcionai/clues" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/path" +) + +// called by export.go to map subcommands to provider-specific handling. +func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, oneDriveExportCmd()) + + c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddOneDriveDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + oneDriveServiceCommand = "onedrive" + oneDriveServiceCommandUseSuffix = "--backup " + + //nolint:lll + oneDriveServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory +corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to current directory +corso export onedrive . --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to my-exports +corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso export onedrive [...] ` +func oneDriveExportCmd() *cobra.Command { + return &cobra.Command{ + Use: oneDriveServiceCommand, + Short: "Export M365 OneDrive service data", + RunE: exportOneDriveCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: oneDriveServiceCommandExportExamples, + } +} + +// processes an onedrive service export. +func exportOneDriveCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeOneDriveOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateOneDriveRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + r, _, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + exportLocation := args[0] + if exportLocation == "" { + // This is unlikely, but adding it just in case. + exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) + } + + Infof(ctx, "Exporting to folder %s", exportLocation) + + sel := utils.IncludeOneDriveRestoreDataSelectors(opts) + utils.FilterOneDriveRestoreInfoSelectors(sel, opts) + + eo, err := r.NewExport( + ctx, + flags.BackupIDFV, + sel.Selector, + utils.MakeExportConfig(ctx, opts.ExportCfg), + ) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to initialize OneDrive export")) + } + + expColl, err := eo.Run(ctx) + if err != nil { + if errors.Is(err, data.ErrNotFound) { + return Only(ctx, clues.New("Backup or backup details missing for id "+flags.BackupIDFV)) + } + + return Only(ctx, clues.Wrap(err, "Failed to run OneDrive export")) + } + + // It would be better to give a progressbar than a spinner, but we + // have know way of knowing how many files are available as of now. + diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") + defer func() { + diskWriteComplete <- struct{}{} + close(diskWriteComplete) + }() + + err = writeExportCollections(ctx, exportLocation, expColl) + if err != nil { + return err + } + + return nil +} + +func writeExportCollections( + ctx context.Context, + exportLocation string, + expColl []export.Collection, +) error { + for _, col := range expColl { + folder := ospath.Join(exportLocation, col.BasePath()) + + for item := range col.Items(ctx) { + err := item.Error + if err != nil { + return Only(ctx, clues.Wrap(err, "getting item").With("dir_name", folder)) + } + + err = writeExportItem(ctx, item, folder) + if err != nil { + return err + } + } + } + + return nil +} + +// writeExportItem writes an ExportItem to disk in the specified folder. +func writeExportItem(ctx context.Context, item export.Item, folder string) error { + name := item.Data.Name + fpath := ospath.Join(folder, name) + + progReader, pclose := observe.ItemSpinner( + ctx, + item.Data.Body, + observe.ItemExportMsg, + clues.Hide(name)) + + defer item.Data.Body.Close() + defer pclose() + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return Only(ctx, clues.Wrap(err, "creating directory").With("dir_name", folder)) + } + + // In case the user tries to restore to a non-clean + // directory, we might run into collisions an fail. + f, err := os.Create(fpath) + if err != nil { + return Only(ctx, clues.Wrap(err, "creating file").With("file_name", name, "file_dir", folder)) + } + + _, err = io.Copy(f, progReader) + if err != nil { + return Only(ctx, clues.Wrap(err, "writing file").With("file_name", name, "file_dir", folder)) + } + + return nil +} diff --git a/src/cli/export/onedrive_test.go b/src/cli/export/onedrive_test.go new file mode 100644 index 000000000..775dd4a70 --- /dev/null +++ b/src/cli/export/onedrive_test.go @@ -0,0 +1,106 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type OneDriveUnitSuite struct { + tester.Suite +} + +func TestOneDriveUnitSuite(t *testing.T) { + suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { + expectUse := oneDriveServiceCommand + " " + oneDriveServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export onedrive", exportCommand, expectUse, oneDriveExportCmd().Short, exportOneDriveCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addOneDriveCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "onedrive", + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + + "--" + flags.ArchiveFN, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + testdata.RestoreDestination, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeOneDriveOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/flags/export.go b/src/cli/flags/export.go new file mode 100644 index 000000000..b9af7e141 --- /dev/null +++ b/src/cli/flags/export.go @@ -0,0 +1,15 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ArchiveFN = "archive" + +var ArchiveFV bool + +// AddExportConfigFlags adds the restore config flag set. +func AddExportConfigFlags(cmd *cobra.Command) { + fs := cmd.Flags() + fs.BoolVar(&ArchiveFV, ArchiveFN, false, "Export data as an archive instead of individual files") +} diff --git a/src/cli/utils/export_config.go b/src/cli/utils/export_config.go new file mode 100644 index 000000000..2fd3827bd --- /dev/null +++ b/src/cli/utils/export_config.go @@ -0,0 +1,38 @@ +package utils + +import ( + "context" + + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/pkg/control" +) + +type ExportCfgOpts struct { + Archive bool + + Populated flags.PopulatedFlags +} + +func makeExportCfgOpts(cmd *cobra.Command) ExportCfgOpts { + return ExportCfgOpts{ + Archive: flags.ArchiveFV, + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} + +func MakeExportConfig( + ctx context.Context, + opts ExportCfgOpts, +) control.ExportConfig { + exportCfg := control.DefaultExportConfig() + + exportCfg.Archive = opts.Archive + + return exportCfg +} diff --git a/src/cli/utils/export_config_test.go b/src/cli/utils/export_config_test.go new file mode 100644 index 000000000..d25d6629b --- /dev/null +++ b/src/cli/utils/export_config_test.go @@ -0,0 +1,54 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" +) + +type ExportCfgUnitSuite struct { + tester.Suite +} + +func TestExportCfgUnitSuite(t *testing.T) { + suite.Run(t, &ExportCfgUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportCfgUnitSuite) TestMakeExportConfig() { + rco := &ExportCfgOpts{Archive: true} + + table := []struct { + name string + populated flags.PopulatedFlags + expect control.ExportConfig + }{ + { + name: "archive populated", + populated: flags.PopulatedFlags{ + flags.ArchiveFN: {}, + }, + expect: control.ExportConfig{ + Archive: true, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := *rco + opts.Populated = test.populated + + result := MakeExportConfig(ctx, opts) + assert.Equal(t, test.expect.Archive, result.Archive) + }) + } +} diff --git a/src/cli/utils/onedrive.go b/src/cli/utils/onedrive.go index 06699adca..e12f35230 100644 --- a/src/cli/utils/onedrive.go +++ b/src/cli/utils/onedrive.go @@ -19,6 +19,7 @@ type OneDriveOpts struct { FileModifiedBefore string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -35,6 +36,7 @@ func MakeOneDriveOpts(cmd *cobra.Command) OneDriveOpts { FileModifiedBefore: flags.FileModifiedBeforeFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/cli/utils/testdata/flags.go b/src/cli/utils/testdata/flags.go index 85131a1e0..b04a9ab63 100644 --- a/src/cli/utils/testdata/flags.go +++ b/src/cli/utils/testdata/flags.go @@ -51,6 +51,8 @@ var ( DeltaPageSize = "deltaPageSize" + Archive = true + AzureClientID = "testAzureClientId" AzureTenantID = "testAzureTenantId" AzureClientSecret = "testAzureClientSecret" @@ -60,4 +62,6 @@ var ( AWSSessionToken = "testAWSSessionToken" CorsoPassphrase = "testCorsoPassphrase" + + RestoreDestination = "test-restore-destination" ) diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index bd94b6531..c05297587 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -133,6 +133,7 @@ func Complete() { const ( ItemBackupMsg = "Backing up item" ItemRestoreMsg = "Restoring item" + ItemExportMsg = "Exporting item" ItemQueueMsg = "Queuing items" ) @@ -281,6 +282,51 @@ func ItemProgress( return bar.ProxyReader(rc), abort } +// ItemSpinner is similar to ItemProgress, but for use in cases where +// we don't know the file size but want to show progress. +func ItemSpinner( + ctx context.Context, + rc io.ReadCloser, + header string, + iname any, +) (io.ReadCloser, func()) { + plain := plainString(iname) + log := logger.Ctx(ctx).With("item", iname) + log.Debug(header) + + if cfg.hidden() || rc == nil { + defer log.Debug("done - " + header) + return rc, func() {} + } + + wg.Add(1) + + barOpts := []mpb.BarOption{ + mpb.PrependDecorators( + decor.Name(header, decor.WCSyncSpaceR), + decor.Name(plain, decor.WCSyncSpaceR), + decor.CurrentKibiByte(" %.1f", decor.WC{W: 8})), + } + + if !cfg.keepBarsAfterComplete { + barOpts = append(barOpts, mpb.BarRemoveOnComplete()) + } + + bar := progress.New(-1, mpb.NopStyle(), barOpts...) + + go waitAndCloseBar(bar, func() { + // might be overly chatty, we can remove if needed. + log.Debug("done - " + header) + })() + + abort := func() { + bar.SetTotal(-1, true) + bar.Abort(true) + } + + return bar.ProxyReader(rc), abort +} + // ProgressWithCount tracks the display of a bar that tracks the completion // of the specified count. // Each write to the provided channel counts as a single increment. diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index c30b7d177..74cf88093 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -16,7 +16,7 @@ import ( ) const ( - defaultRestoreLocation = "Corso_Restore_" + DefaultRestoreLocation = "Corso_Restore_" ) // CollisionPolicy describes how the datalayer behaves in case of a collision. @@ -70,12 +70,12 @@ type RestoreConfig struct { func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { return RestoreConfig{ OnCollision: Skip, - Location: defaultRestoreLocation + dttm.FormatNow(timeFormat), + Location: DefaultRestoreLocation + dttm.FormatNow(timeFormat), } } func DefaultRestoreContainerName(timeFormat dttm.TimeFormat) string { - return defaultRestoreLocation + dttm.FormatNow(timeFormat) + return DefaultRestoreLocation + dttm.FormatNow(timeFormat) } // EnsureRestoreConfigDefaults sets all non-supported values in the config From c654dfba1b107330aae564647e148dfec02e5ec9 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 14:10:35 +0530 Subject: [PATCH 04/17] Export data from SharePoint (#3824) This borrows a lot of the core logic from OneDrive as the internal structure is mostly the same. Prev: https://github.com/alcionai/corso/pull/3822 --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * fixes https://github.com/alcionai/corso/issues/3823 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 2 +- src/cli/export/export.go | 69 ++++++++++ src/cli/export/onedrive.go | 121 +----------------- src/cli/export/onedrive_test.go | 6 +- src/cli/export/sharepoint.go | 100 +++++++++++++++ src/cli/export/sharepoint_test.go | 118 +++++++++++++++++ src/cli/restore/sharepoint_test.go | 2 +- src/cli/utils/sharepoint.go | 2 + src/internal/m365/export.go | 3 +- src/pkg/export/consume.go | 79 ++++++++++++ .../export/consume_test.go} | 20 +-- src/pkg/export/export.go | 4 +- 12 files changed, 389 insertions(+), 137 deletions(-) create mode 100644 src/cli/export/sharepoint.go create mode 100644 src/cli/export/sharepoint_test.go create mode 100644 src/pkg/export/consume.go rename src/{cli/export/export_test.go => pkg/export/consume_test.go} (86%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5170dc50f..906267535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Restore requires the protected resource to have access to the service being restored. ### Added -- Added option to export data from OneDrive backups as individual files or as a single zip file. +- Added option to export data from OneDrive and SharePoint backups as individual files or as a single zip file. ## [v0.11.1] (beta) - 2023-07-20 diff --git a/src/cli/export/export.go b/src/cli/export/export.go index d3ec09a8a..e0deed014 100644 --- a/src/cli/export/export.go +++ b/src/cli/export/export.go @@ -1,11 +1,26 @@ package export import ( + "context" + "errors" + + "github.com/alcionai/clues" "github.com/spf13/cobra" + + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/selectors" ) var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ addOneDriveCommands, + addSharePointCommands, } // AddCommands attaches all `corso export * *` commands to the parent. @@ -37,3 +52,57 @@ func exportCmd() *cobra.Command { func handleExportCmd(cmd *cobra.Command, args []string) error { return cmd.Help() } + +func runExport( + ctx context.Context, + cmd *cobra.Command, + args []string, + ueco utils.ExportCfgOpts, + sel selectors.Selector, + backupID, serviceName string, +) error { + r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + exportLocation := args[0] + if len(exportLocation) == 0 { + // This should not be possible, but adding it just in case. + exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) + } + + Infof(ctx, "Exporting to folder %s", exportLocation) + + eo, err := r.NewExport( + ctx, + backupID, + sel, + utils.MakeExportConfig(ctx, ueco)) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export")) + } + + expColl, err := eo.Run(ctx) + if err != nil { + if errors.Is(err, data.ErrNotFound) { + return Only(ctx, clues.New("Backup or backup details missing for id "+backupID)) + } + + return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export")) + } + + // It would be better to give a progressbar than a spinner, but we + // have any way of knowing how many files are available as of now. + diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") + defer close(diskWriteComplete) + + err = export.ConsumeExportCollections(ctx, exportLocation, expColl, eo.Errors) + if err != nil { + return Only(ctx, err) + } + + return nil +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go index 6e715153d..593149bd9 100644 --- a/src/cli/export/onedrive.go +++ b/src/cli/export/onedrive.go @@ -1,26 +1,12 @@ package export import ( - "context" - "io" - "os" - ospath "path" - - "github.com/alcionai/clues" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/alcionai/corso/src/cli/flags" - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cli/repo" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/observe" - "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/export" - "github.com/alcionai/corso/src/pkg/path" ) // called by export.go to map subcommands to provider-specific handling. @@ -103,113 +89,8 @@ func exportOneDriveCmd(cmd *cobra.Command, args []string) error { return err } - r, _, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) - if err != nil { - return Only(ctx, err) - } - - defer utils.CloseRepo(ctx, r) - - exportLocation := args[0] - if exportLocation == "" { - // This is unlikely, but adding it just in case. - exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) - } - - Infof(ctx, "Exporting to folder %s", exportLocation) - sel := utils.IncludeOneDriveRestoreDataSelectors(opts) utils.FilterOneDriveRestoreInfoSelectors(sel, opts) - eo, err := r.NewExport( - ctx, - flags.BackupIDFV, - sel.Selector, - utils.MakeExportConfig(ctx, opts.ExportCfg), - ) - if err != nil { - return Only(ctx, clues.Wrap(err, "Failed to initialize OneDrive export")) - } - - expColl, err := eo.Run(ctx) - if err != nil { - if errors.Is(err, data.ErrNotFound) { - return Only(ctx, clues.New("Backup or backup details missing for id "+flags.BackupIDFV)) - } - - return Only(ctx, clues.Wrap(err, "Failed to run OneDrive export")) - } - - // It would be better to give a progressbar than a spinner, but we - // have know way of knowing how many files are available as of now. - diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") - defer func() { - diskWriteComplete <- struct{}{} - close(diskWriteComplete) - }() - - err = writeExportCollections(ctx, exportLocation, expColl) - if err != nil { - return err - } - - return nil -} - -func writeExportCollections( - ctx context.Context, - exportLocation string, - expColl []export.Collection, -) error { - for _, col := range expColl { - folder := ospath.Join(exportLocation, col.BasePath()) - - for item := range col.Items(ctx) { - err := item.Error - if err != nil { - return Only(ctx, clues.Wrap(err, "getting item").With("dir_name", folder)) - } - - err = writeExportItem(ctx, item, folder) - if err != nil { - return err - } - } - } - - return nil -} - -// writeExportItem writes an ExportItem to disk in the specified folder. -func writeExportItem(ctx context.Context, item export.Item, folder string) error { - name := item.Data.Name - fpath := ospath.Join(folder, name) - - progReader, pclose := observe.ItemSpinner( - ctx, - item.Data.Body, - observe.ItemExportMsg, - clues.Hide(name)) - - defer item.Data.Body.Close() - defer pclose() - - err := os.MkdirAll(folder, os.ModePerm) - if err != nil { - return Only(ctx, clues.Wrap(err, "creating directory").With("dir_name", folder)) - } - - // In case the user tries to restore to a non-clean - // directory, we might run into collisions an fail. - f, err := os.Create(fpath) - if err != nil { - return Only(ctx, clues.Wrap(err, "creating file").With("file_name", name, "file_dir", folder)) - } - - _, err = io.Copy(f, progReader) - if err != nil { - return Only(ctx, clues.Wrap(err, "writing file").With("file_name", name, "file_dir", folder)) - } - - return nil + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "OneDrive") } diff --git a/src/cli/export/onedrive_test.go b/src/cli/export/onedrive_test.go index 775dd4a70..59ab966e8 100644 --- a/src/cli/export/onedrive_test.go +++ b/src/cli/export/onedrive_test.go @@ -59,6 +59,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { cmd.SetArgs([]string{ "onedrive", + testdata.RestoreDestination, "--" + flags.RunModeFN, flags.RunModeFlagTest, "--" + flags.BackupFN, testdata.BackupInput, "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), @@ -68,15 +69,14 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, - "--" + flags.ArchiveFN, - "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, - testdata.RestoreDestination, + // bool flags + "--" + flags.ArchiveFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output diff --git a/src/cli/export/sharepoint.go b/src/cli/export/sharepoint.go new file mode 100644 index 000000000..ec71a5f2b --- /dev/null +++ b/src/cli/export/sharepoint.go @@ -0,0 +1,100 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addSharePointCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, sharePointExportCmd()) + + c.Use = c.Use + " " + sharePointServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddSharePointDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + sharePointServiceCommand = "sharepoint" + sharePointServiceCommandUseSuffix = "--backup " + + //nolint:lll + sharePointServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's latest backup (1234abcd...) to my-exports directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef my-exports + +# Export files named "ServerRenderTemplate.xsl" in the folder "Display Templates/Style Sheets". as archive to current directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" --archive . + +# Export all files in the folder "Display Templates/Style Sheets" that were created before 2020 to my-exports directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --file-created-before 2020-01-01T00:00:00 --folder "Display Templates/Style Sheets" my-exports + +# Export all files in the "Documents" library to current directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --library Documents --folder "Display Templates/Style Sheets" .` +) + +// `corso export sharepoint [...] ` +func sharePointExportCmd() *cobra.Command { + return &cobra.Command{ + Use: sharePointServiceCommand, + Short: "Export M365 SharePoint service data", + RunE: exportSharePointCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: sharePointServiceCommandExportExamples, + } +} + +// processes an sharepoint service export. +func exportSharePointCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeSharePointOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateSharePointRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) + utils.FilterSharePointRestoreInfoSelectors(sel, opts) + + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "SharePoint") +} diff --git a/src/cli/export/sharepoint_test.go b/src/cli/export/sharepoint_test.go new file mode 100644 index 000000000..48ce28f5c --- /dev/null +++ b/src/cli/export/sharepoint_test.go @@ -0,0 +1,118 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type SharePointUnitSuite struct { + tester.Suite +} + +func TestSharePointUnitSuite(t *testing.T) { + suite.Run(t, &SharePointUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharePointUnitSuite) TestAddSharePointCommands() { + expectUse := sharePointServiceCommand + " " + sharePointServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export sharepoint", exportCommand, expectUse, sharePointExportCmd().Short, exportSharePointCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addSharePointCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "sharepoint", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.LibraryFN, testdata.LibraryInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + "--" + flags.ListItemFN, testdata.FlgInputs(testdata.ListItemInput), + "--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput), + "--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput), + "--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput), + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeSharePointOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.LibraryInput, opts.Library) + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.ElementsMatch(t, testdata.ListItemInput, opts.ListItem) + assert.ElementsMatch(t, testdata.ListFolderInput, opts.ListFolder) + + assert.ElementsMatch(t, testdata.PageInput, opts.Page) + assert.ElementsMatch(t, testdata.PageFolderInput, opts.PageFolder) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/restore/sharepoint_test.go b/src/cli/restore/sharepoint_test.go index 6a8de8e57..c9bc8277f 100644 --- a/src/cli/restore/sharepoint_test.go +++ b/src/cli/restore/sharepoint_test.go @@ -34,7 +34,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { expectShort string expectRunE func(*cobra.Command, []string) error }{ - {"restore onedrive", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, + {"restore sharepoint", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/sharepoint.go b/src/cli/utils/sharepoint.go index 6672beda1..2ab43d90c 100644 --- a/src/cli/utils/sharepoint.go +++ b/src/cli/utils/sharepoint.go @@ -32,6 +32,7 @@ type SharePointOpts struct { Page []string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -56,6 +57,7 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts { PageFolder: flags.PageFolderFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 085881803..4da037e26 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -41,7 +41,8 @@ func (ctrl *Controller) ProduceExportCollections( ) switch sels.Service { - case selectors.ServiceOneDrive: + case selectors.ServiceOneDrive, selectors.ServiceSharePoint: + // OneDrive and SharePoint can share the code to create collections expCollections, err = onedrive.ProduceExportCollections( ctx, backupVersion, diff --git a/src/pkg/export/consume.go b/src/pkg/export/consume.go new file mode 100644 index 000000000..899f9c3ba --- /dev/null +++ b/src/pkg/export/consume.go @@ -0,0 +1,79 @@ +package export + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/fault" +) + +func ConsumeExportCollections( + ctx context.Context, + exportLocation string, + expColl []Collection, + errs *fault.Bus, +) error { + el := errs.Local() + + for _, col := range expColl { + if el.Failure() != nil { + break + } + + folder := filepath.Join(exportLocation, col.BasePath()) + ictx := clues.Add(ctx, "dir_name", folder) + + for item := range col.Items(ctx) { + if item.Error != nil { + el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item").WithClues(ctx)) + } + + if err := writeItem(ictx, item, folder); err != nil { + el.AddRecoverable( + ictx, + clues.Wrap(err, "writing item").With("file_name", item.Data.Name).WithClues(ctx)) + } + } + } + + return el.Failure() +} + +// writeItem writes an ExportItem to disk in the specified folder. +func writeItem(ctx context.Context, item Item, folder string) error { + name := item.Data.Name + fpath := filepath.Join(folder, name) + + progReader, pclose := observe.ItemSpinner( + ctx, + item.Data.Body, + observe.ItemExportMsg, + clues.Hide(name)) + + defer item.Data.Body.Close() + defer pclose() + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return clues.Wrap(err, "creating directory") + } + + // In case the user tries to restore to a non-clean + // directory, we might run into collisions an fail. + f, err := os.Create(fpath) + if err != nil { + return clues.Wrap(err, "creating file") + } + + _, err = io.Copy(f, progReader) + if err != nil { + return clues.Wrap(err, "writing data") + } + + return nil +} diff --git a/src/cli/export/export_test.go b/src/pkg/export/consume_test.go similarity index 86% rename from src/cli/export/export_test.go rename to src/pkg/export/consume_test.go index f3df68177..7d22dc237 100644 --- a/src/cli/export/export_test.go +++ b/src/pkg/export/consume_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" ) type ExportE2ESuite struct { @@ -31,12 +31,12 @@ func (suite *ExportE2ESuite) SetupSuite() { type mockExportCollection struct { path string - items []export.Item + items []Item } func (mec mockExportCollection) BasePath() string { return mec.path } -func (mec mockExportCollection) Items(context.Context) <-chan export.Item { - ch := make(chan export.Item) +func (mec mockExportCollection) Items(context.Context) <-chan Item { + ch := make(chan Item) go func() { defer close(ch) @@ -49,7 +49,7 @@ func (mec mockExportCollection) Items(context.Context) <-chan export.Item { return ch } -func (suite *ExportE2ESuite) TestWriteExportCollection() { +func (suite *ExportE2ESuite) TestConsumeExportCollection() { type ei struct { name string body string @@ -132,12 +132,12 @@ func (suite *ExportE2ESuite) TestWriteExportCollection() { ctx, flush := tester.NewContext(t) defer flush() - ecs := []export.Collection{} + ecs := []Collection{} for _, col := range test.cols { - items := []export.Item{} + items := []Item{} for _, item := range col.items { - items = append(items, export.Item{ - Data: export.ItemData{ + items = append(items, Item{ + Data: ItemData{ Name: item.name, Body: io.NopCloser((bytes.NewBufferString(item.body))), }, @@ -154,7 +154,7 @@ func (suite *ExportE2ESuite) TestWriteExportCollection() { require.NoError(t, err) defer os.RemoveAll(dir) - err = writeExportCollections(ctx, dir, ecs) + err = ConsumeExportCollections(ctx, dir, ecs, fault.New(true)) require.NoError(t, err, "writing data") for _, col := range test.cols { diff --git a/src/pkg/export/export.go b/src/pkg/export/export.go index 76a6b6d8b..73c173e04 100644 --- a/src/pkg/export/export.go +++ b/src/pkg/export/export.go @@ -7,7 +7,9 @@ import ( // Collection is the interface that is returned to the SDK consumer type Collection interface { - // BasePath gets the base path of the collection + // BasePath gets the base path of the collection. This is derived + // from FullPath, but trim out thing like drive id or any other part + // that is not needed to show the path to the collection. BasePath() string // Items gets the items within the collection(folder) From 26d50832695743077e053338fc14ce3241197dba Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 15:20:38 +0530 Subject: [PATCH 05/17] Separate pieces of sanity test into different package (#3909) Just moving around code, no changes. This just splits the single fine sanity test into multiple files. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/sanity_test/restore/exchange.go | 219 +++++++ src/cmd/sanity_test/restore/onedrive.go | 353 +++++++++++ src/cmd/sanity_test/restore/sharepoint.go | 39 ++ src/cmd/sanity_test/sanity_tests.go | 686 +--------------------- src/cmd/sanity_test/utils/utils.go | 82 +++ 5 files changed, 701 insertions(+), 678 deletions(-) create mode 100644 src/cmd/sanity_test/restore/exchange.go create mode 100644 src/cmd/sanity_test/restore/onedrive.go create mode 100644 src/cmd/sanity_test/restore/sharepoint.go create mode 100644 src/cmd/sanity_test/utils/utils.go diff --git a/src/cmd/sanity_test/restore/exchange.go b/src/cmd/sanity_test/restore/exchange.go new file mode 100644 index 000000000..e484550de --- /dev/null +++ b/src/cmd/sanity_test/restore/exchange.go @@ -0,0 +1,219 @@ +package restore + +import ( + "context" + "fmt" + stdpath "path" + "strings" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/users" + + "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/filters" +) + +// CheckEmailRestoration verifies that the emails count in restored folder is equivalent to +// emails in actual m365 account +func CheckEmailRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + testUser, folderName, dataFolder, baseBackupFolder string, + startTime time.Time, +) { + var ( + restoreFolder models.MailFolderable + itemCount = make(map[string]int32) + restoreItemCount = make(map[string]int32) + builder = client.Users().ByUserId(testUser).MailFolders() + ) + + for { + result, err := builder.Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting mail folders", err) + } + + values := result.GetValue() + + for _, v := range values { + itemName := ptr.Val(v.GetDisplayName()) + + if itemName == folderName { + restoreFolder = v + continue + } + + if itemName == dataFolder || itemName == baseBackupFolder { + // otherwise, recursively aggregate all child folders. + getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount) + + itemCount[itemName] = ptr.Val(v.GetTotalItemCount()) + } + } + + link, ok := ptr.ValOK(result.GetOdataNextLink()) + if !ok { + break + } + + builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter()) + } + + folderID := ptr.Val(restoreFolder.GetId()) + folderName = ptr.Val(restoreFolder.GetDisplayName()) + ctx = clues.Add( + ctx, + "restore_folder_id", folderID, + "restore_folder_name", folderName) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting restore folder child folders", err) + } + + for _, fld := range childFolder.GetValue() { + restoreDisplayName := ptr.Val(fld.GetDisplayName()) + + // check if folder is the data folder we loaded or the base backup to verify + // the incremental backup worked fine + if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) { + count, _ := ptr.ValOK(fld.GetTotalItemCount()) + + restoreItemCount[restoreDisplayName] = count + checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount) + } + } + + verifyEmailData(ctx, restoreItemCount, itemCount) +} + +func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) { + for fldName, expected := range messageCount { + got := restoreMessageCount[fldName] + + utils.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("Restore item counts do not match: %s", fldName), + expected, + got) + } +} + +// getAllSubFolder will recursively check for all subfolders and get the corresponding +// email count. +func getAllMailSubFolders( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + testUser string, + r models.MailFolderable, + parentFolder, + dataFolder string, + messageCount map[string]int32, +) { + var ( + folderID = ptr.Val(r.GetId()) + count int32 = 99 + options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Top: &count, + }, + } + ) + + ctx = clues.Add(ctx, "parent_folder_id", folderID) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, options) + if err != nil { + utils.Fatal(ctx, "getting mail subfolders", err) + } + + for _, child := range childFolder.GetValue() { + var ( + childDisplayName = ptr.Val(child.GetDisplayName()) + childFolderCount = ptr.Val(child.GetChildFolderCount()) + //nolint:forbidigo + fullFolderName = stdpath.Join(parentFolder, childDisplayName) + ) + + if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { + messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount()) + // recursively check for subfolders + if childFolderCount > 0 { + parentFolder := fullFolderName + + getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount) + } + } + } +} + +// checkAllSubFolder will recursively traverse inside the restore folder and +// verify that data matched in all subfolders +func checkAllSubFolder( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + r models.MailFolderable, + testUser, + parentFolder, + dataFolder string, + restoreMessageCount map[string]int32, +) { + var ( + folderID = ptr.Val(r.GetId()) + count int32 = 99 + options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Top: &count, + }, + } + ) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, options) + if err != nil { + utils.Fatal(ctx, "getting mail subfolders", err) + } + + for _, child := range childFolder.GetValue() { + var ( + childDisplayName = ptr.Val(child.GetDisplayName()) + //nolint:forbidigo + fullFolderName = stdpath.Join(parentFolder, childDisplayName) + ) + + if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { + childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount()) + restoreMessageCount[fullFolderName] = childTotalCount + } + + childFolderCount := ptr.Val(child.GetChildFolderCount()) + + if childFolderCount > 0 { + parentFolder := fullFolderName + checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount) + } + } +} diff --git a/src/cmd/sanity_test/restore/onedrive.go b/src/cmd/sanity_test/restore/onedrive.go new file mode 100644 index 000000000..ca332d608 --- /dev/null +++ b/src/cmd/sanity_test/restore/onedrive.go @@ -0,0 +1,353 @@ +package restore + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/path" +) + +const ( + owner = "owner" +) + +type permissionInfo struct { + entityID string + roles []string +} + +func CheckOneDriveRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + userID, folderName, dataFolder string, + startTime time.Time, +) { + drive, err := client. + Users(). + ByUserId(userID). + Drive(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting the drive:", err) + } + + checkDriveRestoration( + ctx, + client, + path.OneDriveService, + folderName, + ptr.Val(drive.GetId()), + ptr.Val(drive.GetName()), + dataFolder, + startTime, + false) +} + +func checkDriveRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + service path.ServiceType, + folderName, + driveID, + driveName, + dataFolder string, + startTime time.Time, + skipPermissionTest bool, +) { + var ( + // map itemID -> item size + fileSizes = make(map[string]int64) + // map itemID -> permission id -> []permission roles + folderPermissions = make(map[string][]permissionInfo) + restoreFile = make(map[string]int64) + restoredFolderPermissions = make(map[string][]permissionInfo) + ) + + var restoreFolderID string + + ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) + + response, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId("root"). + Children(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting drive by id", err) + } + + for _, driveItem := range response.GetValue() { + var ( + itemID = ptr.Val(driveItem.GetId()) + itemName = ptr.Val(driveItem.GetName()) + ) + + if itemName == folderName { + restoreFolderID = itemID + continue + } + + if itemName != dataFolder { + utils.LogAndPrint(ctx, "test data for folder: %s", dataFolder) + continue + } + + // if it's a file check the size + if driveItem.GetFile() != nil { + fileSizes[itemName] = ptr.Val(driveItem.GetSize()) + } + + if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { + continue + } + + // currently we don't restore blank folders. + // skip permission check for empty folders + if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { + utils.LogAndPrint(ctx, "skipped empty folder: %s", itemName) + continue + } + + folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) + } + + getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) + + checkRestoredDriveItemPermissions( + ctx, + service, + skipPermissionTest, + folderPermissions, + restoredFolderPermissions) + + for fileName, expected := range fileSizes { + utils.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := restoreFile[fileName] + + utils.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} + +func checkRestoredDriveItemPermissions( + ctx context.Context, + service path.ServiceType, + skip bool, + folderPermissions map[string][]permissionInfo, + restoredFolderPermissions map[string][]permissionInfo, +) { + if skip { + return + } + + /** + TODO: replace this check with testElementsMatch + from internal/connecter/graph_connector_helper_test.go + **/ + + for folderName, permissions := range folderPermissions { + utils.LogAndPrint(ctx, "checking for folder: %s", folderName) + + restoreFolderPerm := restoredFolderPermissions[folderName] + + if len(permissions) < 1 { + utils.LogAndPrint(ctx, "no permissions found in: %s", folderName) + continue + } + + permCheck := func() bool { return len(permissions) == len(restoreFolderPerm) } + + if service == path.SharePointService { + permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } + } + + utils.Assert( + ctx, + permCheck, + fmt.Sprintf("wrong number of restored permissions: %s", folderName), + permissions, + restoreFolderPerm) + + for _, perm := range permissions { + eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } + i := slices.IndexFunc(restoreFolderPerm, eqID) + + utils.Assert( + ctx, + func() bool { return i >= 0 }, + fmt.Sprintf("permission was restored in: %s", folderName), + perm.entityID, + restoreFolderPerm) + + // permissions should be sorted, so a by-index comparison works + restored := restoreFolderPerm[i] + + utils.Assert( + ctx, + func() bool { return slices.Equal(perm.roles, restored.roles) }, + fmt.Sprintf("different roles restored: %s", folderName), + perm.roles, + restored.roles) + } + } +} + +func getOneDriveChildFolder( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, itemID, parentName string, + fileSizes map[string]int64, + folderPermission map[string][]permissionInfo, + startTime time.Time, +) { + response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting child folder", err) + } + + for _, driveItem := range response.GetValue() { + var ( + itemID = ptr.Val(driveItem.GetId()) + itemName = ptr.Val(driveItem.GetName()) + fullName = parentName + "/" + itemName + ) + + folderTime, hasTime := utils.MustGetTimeFromName(ctx, itemName) + if !utils.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { + continue + } + + // if it's a file check the size + if driveItem.GetFile() != nil { + fileSizes[fullName] = ptr.Val(driveItem.GetSize()) + } + + if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { + continue + } + + // currently we don't restore blank folders. + // skip permission check for empty folders + if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { + utils.LogAndPrint(ctx, "skipped empty folder: %s", fullName) + + continue + } + + folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime) + } +} + +func getRestoredDrive( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, restoreFolderID string, + restoreFile map[string]int64, + restoreFolder map[string][]permissionInfo, + startTime time.Time, +) { + restored, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(restoreFolderID). + Children(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting child folder", err) + } + + for _, item := range restored.GetValue() { + var ( + itemID = ptr.Val(item.GetId()) + itemName = ptr.Val(item.GetName()) + itemSize = ptr.Val(item.GetSize()) + ) + + if item.GetFile() != nil { + restoreFile[itemName] = itemSize + continue + } + + if item.GetFolder() == nil && item.GetPackage() == nil { + continue + } + + restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime) + } +} + +// --------------------------------------------------------------------------- +// permission helpers +// --------------------------------------------------------------------------- + +func permissionIn( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, itemID string, +) []permissionInfo { + pi := []permissionInfo{} + + pcr, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(itemID). + Permissions(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting permission", err) + } + + for _, perm := range pcr.GetValue() { + if perm.GetGrantedToV2() == nil { + continue + } + + var ( + gv2 = perm.GetGrantedToV2() + permInfo = permissionInfo{} + entityID string + ) + + // TODO: replace with filterUserPermissions in onedrive item.go + if gv2.GetUser() != nil { + entityID = ptr.Val(gv2.GetUser().GetId()) + } else if gv2.GetGroup() != nil { + entityID = ptr.Val(gv2.GetGroup().GetId()) + } + + roles := utils.FilterSlice(perm.GetRoles(), owner) + for _, role := range roles { + permInfo.entityID = entityID + permInfo.roles = append(permInfo.roles, role) + } + + if len(roles) > 0 { + slices.Sort(permInfo.roles) + pi = append(pi, permInfo) + } + } + + return pi +} diff --git a/src/cmd/sanity_test/restore/sharepoint.go b/src/cmd/sanity_test/restore/sharepoint.go new file mode 100644 index 000000000..d84e28f6c --- /dev/null +++ b/src/cmd/sanity_test/restore/sharepoint.go @@ -0,0 +1,39 @@ +package restore + +import ( + "context" + "time" + + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/path" +) + +func CheckSharePointRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + siteID, userID, folderName, dataFolder string, + startTime time.Time, +) { + drive, err := client. + Sites(). + BySiteId(siteID). + Drive(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting the drive:", err) + } + + checkDriveRestoration( + ctx, + client, + path.SharePointService, + folderName, + ptr.Val(drive.GetId()), + ptr.Val(drive.GetName()), + dataFolder, + startTime, + true) +} diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index ab89586b4..a4d455edd 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -2,45 +2,20 @@ package main import ( "context" - "errors" - "fmt" "os" - stdpath "path" "strings" "time" "github.com/alcionai/clues" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/users" - "golang.org/x/exp/slices" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/cmd/sanity_test/utils" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester/tconfig" - "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/logger" - "github.com/alcionai/corso/src/pkg/path" ) -// --------------------------------------------------------------------------- -// types, consts, etc -// --------------------------------------------------------------------------- - -type permissionInfo struct { - entityID string - roles []string -} - -const ( - owner = "owner" -) - -// --------------------------------------------------------------------------- -// main -// --------------------------------------------------------------------------- - func main() { ls := logger.Settings{ File: logger.GetLogFile(""), @@ -60,7 +35,7 @@ func main() { os.Getenv("AZURE_CLIENT_ID"), os.Getenv("AZURE_CLIENT_SECRET")) if err != nil { - fatal(ctx, "creating adapter", err) + utils.Fatal(ctx, "creating adapter", err) } var ( @@ -69,7 +44,7 @@ func main() { testSite = tconfig.GetM365SiteID(ctx) testService = os.Getenv("SANITY_RESTORE_SERVICE") folder = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER")) - startTime, _ = mustGetTimeFromName(ctx, folder) + startTime, _ = utils.MustGetTimeFromName(ctx, folder) dataFolder = os.Getenv("TEST_DATA") baseBackupFolder = os.Getenv("BASE_BACKUP") ) @@ -85,657 +60,12 @@ func main() { switch testService { case "exchange": - checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) + restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) case "onedrive": - checkOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) + restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) case "sharepoint": - checkSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) default: - fatal(ctx, "no service specified", nil) + utils.Fatal(ctx, "unknown service for restore sanity tests", nil) } } - -// --------------------------------------------------------------------------- -// exchange -// --------------------------------------------------------------------------- - -// checkEmailRestoration verifies that the emails count in restored folder is equivalent to -// emails in actual m365 account -func checkEmailRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - testUser, folderName, dataFolder, baseBackupFolder string, - startTime time.Time, -) { - var ( - restoreFolder models.MailFolderable - itemCount = make(map[string]int32) - restoreItemCount = make(map[string]int32) - builder = client.Users().ByUserId(testUser).MailFolders() - ) - - for { - result, err := builder.Get(ctx, nil) - if err != nil { - fatal(ctx, "getting mail folders", err) - } - - values := result.GetValue() - - for _, v := range values { - itemName := ptr.Val(v.GetDisplayName()) - - if itemName == folderName { - restoreFolder = v - continue - } - - if itemName == dataFolder || itemName == baseBackupFolder { - // otherwise, recursively aggregate all child folders. - getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount) - - itemCount[itemName] = ptr.Val(v.GetTotalItemCount()) - } - } - - link, ok := ptr.ValOK(result.GetOdataNextLink()) - if !ok { - break - } - - builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter()) - } - - folderID := ptr.Val(restoreFolder.GetId()) - folderName = ptr.Val(restoreFolder.GetDisplayName()) - ctx = clues.Add( - ctx, - "restore_folder_id", folderID, - "restore_folder_name", folderName) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting restore folder child folders", err) - } - - for _, fld := range childFolder.GetValue() { - restoreDisplayName := ptr.Val(fld.GetDisplayName()) - - // check if folder is the data folder we loaded or the base backup to verify - // the incremental backup worked fine - if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) { - count, _ := ptr.ValOK(fld.GetTotalItemCount()) - - restoreItemCount[restoreDisplayName] = count - checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount) - } - } - - verifyEmailData(ctx, restoreItemCount, itemCount) -} - -func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) { - for fldName, expected := range messageCount { - got := restoreMessageCount[fldName] - - assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("Restore item counts do not match: %s", fldName), - expected, - got) - } -} - -// getAllSubFolder will recursively check for all subfolders and get the corresponding -// email count. -func getAllMailSubFolders( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - testUser string, - r models.MailFolderable, - parentFolder, - dataFolder string, - messageCount map[string]int32, -) { - var ( - folderID = ptr.Val(r.GetId()) - count int32 = 99 - options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Top: &count, - }, - } - ) - - ctx = clues.Add(ctx, "parent_folder_id", folderID) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, options) - if err != nil { - fatal(ctx, "getting mail subfolders", err) - } - - for _, child := range childFolder.GetValue() { - var ( - childDisplayName = ptr.Val(child.GetDisplayName()) - childFolderCount = ptr.Val(child.GetChildFolderCount()) - //nolint:forbidigo - fullFolderName = stdpath.Join(parentFolder, childDisplayName) - ) - - if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { - messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount()) - // recursively check for subfolders - if childFolderCount > 0 { - parentFolder := fullFolderName - - getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount) - } - } - } -} - -// checkAllSubFolder will recursively traverse inside the restore folder and -// verify that data matched in all subfolders -func checkAllSubFolder( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - r models.MailFolderable, - testUser, - parentFolder, - dataFolder string, - restoreMessageCount map[string]int32, -) { - var ( - folderID = ptr.Val(r.GetId()) - count int32 = 99 - options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Top: &count, - }, - } - ) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, options) - if err != nil { - fatal(ctx, "getting mail subfolders", err) - } - - for _, child := range childFolder.GetValue() { - var ( - childDisplayName = ptr.Val(child.GetDisplayName()) - //nolint:forbidigo - fullFolderName = stdpath.Join(parentFolder, childDisplayName) - ) - - if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { - childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount()) - restoreMessageCount[fullFolderName] = childTotalCount - } - - childFolderCount := ptr.Val(child.GetChildFolderCount()) - - if childFolderCount > 0 { - parentFolder := fullFolderName - checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount) - } - } -} - -// --------------------------------------------------------------------------- -// oneDrive -// --------------------------------------------------------------------------- - -func checkOneDriveRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - userID, folderName, dataFolder string, - startTime time.Time, -) { - drive, err := client. - Users(). - ByUserId(userID). - Drive(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting the drive:", err) - } - - checkDriveRestoration( - ctx, - client, - path.OneDriveService, - folderName, - ptr.Val(drive.GetId()), - ptr.Val(drive.GetName()), - dataFolder, - startTime, - false) -} - -// --------------------------------------------------------------------------- -// sharePoint -// --------------------------------------------------------------------------- - -func checkSharePointRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - siteID, userID, folderName, dataFolder string, - startTime time.Time, -) { - drive, err := client. - Sites(). - BySiteId(siteID). - Drive(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting the drive:", err) - } - - checkDriveRestoration( - ctx, - client, - path.SharePointService, - folderName, - ptr.Val(drive.GetId()), - ptr.Val(drive.GetName()), - dataFolder, - startTime, - true) -} - -// --------------------------------------------------------------------------- -// shared drive tests -// --------------------------------------------------------------------------- - -func checkDriveRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - service path.ServiceType, - folderName, - driveID, - driveName, - dataFolder string, - startTime time.Time, - skipPermissionTest bool, -) { - var ( - // map itemID -> item size - fileSizes = make(map[string]int64) - // map itemID -> permission id -> []permission roles - folderPermissions = make(map[string][]permissionInfo) - restoreFile = make(map[string]int64) - restoredFolderPermissions = make(map[string][]permissionInfo) - ) - - var restoreFolderID string - - ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) - - response, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId("root"). - Children(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting drive by id", err) - } - - for _, driveItem := range response.GetValue() { - var ( - itemID = ptr.Val(driveItem.GetId()) - itemName = ptr.Val(driveItem.GetName()) - ) - - if itemName == folderName { - restoreFolderID = itemID - continue - } - - if itemName != dataFolder { - logAndPrint(ctx, "test data for folder: %s", dataFolder) - continue - } - - // if it's a file check the size - if driveItem.GetFile() != nil { - fileSizes[itemName] = ptr.Val(driveItem.GetSize()) - } - - if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { - continue - } - - // currently we don't restore blank folders. - // skip permission check for empty folders - if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - logAndPrint(ctx, "skipped empty folder: %s", itemName) - continue - } - - folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) - } - - getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) - - checkRestoredDriveItemPermissions( - ctx, - service, - skipPermissionTest, - folderPermissions, - restoredFolderPermissions) - - for fileName, expected := range fileSizes { - logAndPrint(ctx, "checking for file: %s", fileName) - - got := restoreFile[fileName] - - assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("different file size: %s", fileName), - expected, - got) - } - - fmt.Println("Success") -} - -func checkRestoredDriveItemPermissions( - ctx context.Context, - service path.ServiceType, - skip bool, - folderPermissions map[string][]permissionInfo, - restoredFolderPermissions map[string][]permissionInfo, -) { - if skip { - return - } - - /** - TODO: replace this check with testElementsMatch - from internal/connecter/graph_connector_helper_test.go - **/ - - for folderName, permissions := range folderPermissions { - logAndPrint(ctx, "checking for folder: %s", folderName) - - restoreFolderPerm := restoredFolderPermissions[folderName] - - if len(permissions) < 1 { - logAndPrint(ctx, "no permissions found in: %s", folderName) - continue - } - - permCheck := func() bool { return len(permissions) == len(restoreFolderPerm) } - - if service == path.SharePointService { - permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } - } - - assert( - ctx, - permCheck, - fmt.Sprintf("wrong number of restored permissions: %s", folderName), - permissions, - restoreFolderPerm) - - for _, perm := range permissions { - eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } - i := slices.IndexFunc(restoreFolderPerm, eqID) - - assert( - ctx, - func() bool { return i >= 0 }, - fmt.Sprintf("permission was restored in: %s", folderName), - perm.entityID, - restoreFolderPerm) - - // permissions should be sorted, so a by-index comparison works - restored := restoreFolderPerm[i] - - assert( - ctx, - func() bool { return slices.Equal(perm.roles, restored.roles) }, - fmt.Sprintf("different roles restored: %s", folderName), - perm.roles, - restored.roles) - } - } -} - -func getOneDriveChildFolder( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, itemID, parentName string, - fileSizes map[string]int64, - folderPermission map[string][]permissionInfo, - startTime time.Time, -) { - response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) - if err != nil { - fatal(ctx, "getting child folder", err) - } - - for _, driveItem := range response.GetValue() { - var ( - itemID = ptr.Val(driveItem.GetId()) - itemName = ptr.Val(driveItem.GetName()) - fullName = parentName + "/" + itemName - ) - - folderTime, hasTime := mustGetTimeFromName(ctx, itemName) - if !isWithinTimeBound(ctx, startTime, folderTime, hasTime) { - continue - } - - // if it's a file check the size - if driveItem.GetFile() != nil { - fileSizes[fullName] = ptr.Val(driveItem.GetSize()) - } - - if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { - continue - } - - // currently we don't restore blank folders. - // skip permission check for empty folders - if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - logAndPrint(ctx, "skipped empty folder: %s", fullName) - - continue - } - - folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime) - } -} - -func getRestoredDrive( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, restoreFolderID string, - restoreFile map[string]int64, - restoreFolder map[string][]permissionInfo, - startTime time.Time, -) { - restored, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId(restoreFolderID). - Children(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting child folder", err) - } - - for _, item := range restored.GetValue() { - var ( - itemID = ptr.Val(item.GetId()) - itemName = ptr.Val(item.GetName()) - itemSize = ptr.Val(item.GetSize()) - ) - - if item.GetFile() != nil { - restoreFile[itemName] = itemSize - continue - } - - if item.GetFolder() == nil && item.GetPackage() == nil { - continue - } - - restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime) - } -} - -// --------------------------------------------------------------------------- -// permission helpers -// --------------------------------------------------------------------------- - -func permissionIn( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, itemID string, -) []permissionInfo { - pi := []permissionInfo{} - - pcr, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId(itemID). - Permissions(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting permission", err) - } - - for _, perm := range pcr.GetValue() { - if perm.GetGrantedToV2() == nil { - continue - } - - var ( - gv2 = perm.GetGrantedToV2() - permInfo = permissionInfo{} - entityID string - ) - - // TODO: replace with filterUserPermissions in onedrive item.go - if gv2.GetUser() != nil { - entityID = ptr.Val(gv2.GetUser().GetId()) - } else if gv2.GetGroup() != nil { - entityID = ptr.Val(gv2.GetGroup().GetId()) - } - - roles := filterSlice(perm.GetRoles(), owner) - for _, role := range roles { - permInfo.entityID = entityID - permInfo.roles = append(permInfo.roles, role) - } - - if len(roles) > 0 { - slices.Sort(permInfo.roles) - pi = append(pi, permInfo) - } - } - - return pi -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -func fatal(ctx context.Context, msg string, err error) { - logger.CtxErr(ctx, err).Error("test failure: " + msg) - fmt.Println(msg+": ", err) - os.Exit(1) -} - -func mustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { - t, err := dttm.ExtractTime(name) - if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { - fatal(ctx, "extracting time from name: "+name, err) - } - - return t, !errors.Is(err, dttm.ErrNoTimeString) -} - -func isWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { - if hasTime { - if bound.Before(check) { - logger.Ctx(ctx). - With("boundary_time", bound, "check_time", check). - Info("skipping restore folder: not older than time bound") - - return false - } - } - - return true -} - -func filterSlice(sl []string, remove string) []string { - r := []string{} - - for _, s := range sl { - if !strings.EqualFold(s, remove) { - r = append(r, s) - } - } - - return r -} - -func assert( - ctx context.Context, - passes func() bool, - header string, - expect, current any, -) { - if passes() { - return - } - - header = "Error: " + header - expected := fmt.Sprintf("* Expected: %+v", expect) - got := fmt.Sprintf("* Current: %+v", current) - - logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) - - fmt.Println(header) - fmt.Println(expected) - fmt.Println(got) - - os.Exit(1) -} - -func logAndPrint(ctx context.Context, tmpl string, vs ...any) { - logger.Ctx(ctx).Infof(tmpl, vs...) - fmt.Printf(tmpl+"\n", vs...) -} diff --git a/src/cmd/sanity_test/utils/utils.go b/src/cmd/sanity_test/utils/utils.go new file mode 100644 index 000000000..d0a877d61 --- /dev/null +++ b/src/cmd/sanity_test/utils/utils.go @@ -0,0 +1,82 @@ +package utils + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/logger" +) + +func Assert( + ctx context.Context, + passes func() bool, + header string, + expect, current any, +) { + if passes() { + return + } + + header = "Error: " + header + expected := fmt.Sprintf("* Expected: %+v", expect) + got := fmt.Sprintf("* Current: %+v", current) + + logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) + + fmt.Println(header) + fmt.Println(expected) + fmt.Println(got) + + os.Exit(1) +} + +func Fatal(ctx context.Context, msg string, err error) { + logger.CtxErr(ctx, err).Error("test failure: " + msg) + fmt.Println(msg+": ", err) + os.Exit(1) +} + +func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { + t, err := dttm.ExtractTime(name) + if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { + Fatal(ctx, "extracting time from name: "+name, err) + } + + return t, !errors.Is(err, dttm.ErrNoTimeString) +} + +func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { + if hasTime { + if bound.Before(check) { + logger.Ctx(ctx). + With("boundary_time", bound, "check_time", check). + Info("skipping restore folder: not older than time bound") + + return false + } + } + + return true +} + +func FilterSlice(sl []string, remove string) []string { + r := []string{} + + for _, s := range sl { + if !strings.EqualFold(s, remove) { + r = append(r, s) + } + } + + return r +} + +func LogAndPrint(ctx context.Context, tmpl string, vs ...any) { + logger.Ctx(ctx).Infof(tmpl, vs...) + fmt.Printf(tmpl+"\n", vs...) +} From 8f30db4f6e4b2cd3c764d24f1902551f7f91f847 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 16:13:59 +0530 Subject: [PATCH 06/17] Sanity tests for OneDrive exports (#3910) Add sanity tests for OneDrive exports (archive and non-archive exports). --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * https://github.com/alcionai/corso/issues/3889 #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- .../actions/backup-restore-test/action.yml | 72 ++++++++- src/cmd/sanity_test/common/common.go | 6 + src/cmd/sanity_test/common/utils.go | 82 ++++++++++ src/cmd/sanity_test/export/onedrive.go | 88 +++++++++++ src/cmd/sanity_test/restore/exchange.go | 12 +- src/cmd/sanity_test/restore/onedrive.go | 144 ++++++++++-------- src/cmd/sanity_test/restore/sharepoint.go | 4 +- src/cmd/sanity_test/sanity_tests.go | 45 ++++-- 8 files changed, 363 insertions(+), 90 deletions(-) create mode 100644 src/cmd/sanity_test/common/common.go create mode 100644 src/cmd/sanity_test/common/utils.go create mode 100644 src/cmd/sanity_test/export/onedrive.go diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 2d161af63..56213d3e7 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -72,12 +72,78 @@ runs: cat /tmp/corsologs - - name: Check ${{ inputs.service }} ${{ inputs.kind }} + - name: Check restore ${{ inputs.service }} ${{ inputs.kind }} shell: bash working-directory: src env: - SANITY_RESTORE_FOLDER: ${{ steps.restore.outputs.result }} - SANITY_RESTORE_SERVICE: ${{ inputs.service }} + SANITY_TEST_KIND: restore + SANITY_TEST_FOLDER: ${{ steps.restore.outputs.result }} + SANITY_TEST_SERVICE: ${{ inputs.service }} + TEST_DATA: ${{ inputs.test-folder }} + BASE_BACKUP: ${{ inputs.base-backup }} + run: | + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log + ./sanity-test + + - name: Export ${{ inputs.service }} ${{ inputs.kind }} + id: export + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + run: | + set -euo pipefail + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log + ./corso export '${{ inputs.service }}' \ + /tmp/export-${{ inputs.service }}-${{inputs.kind }} \ + --no-stats \ + --hide-progress \ + ${{ inputs.export-args }} \ + --backup '${{ steps.backup.outputs.result }}' + + cat /tmp/corsologs + + - name: Check export ${{ inputs.service }} ${{ inputs.kind }} + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} + env: + SANITY_TEST_KIND: export + SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }} + SANITY_TEST_SERVICE: ${{ inputs.service }} + TEST_DATA: ${{ inputs.test-folder }} + BASE_BACKUP: ${{ inputs.base-backup }} + run: | + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log + ./sanity-test + + - name: Export archive ${{ inputs.service }} ${{ inputs.kind }} + id: export-archive + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + run: | + set -euo pipefail + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log + ./corso export '${{ inputs.service }}' \ + /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \ + --no-stats \ + --hide-progress \ + --archive \ + ${{ inputs.export-args }} \ + --backup '${{ steps.backup.outputs.result }}' + + unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \ + -d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + cat /tmp/corsologs + + - name: Check archive export ${{ inputs.service }} ${{ inputs.kind }} + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} + env: + SANITY_TEST_KIND: export + SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + SANITY_TEST_SERVICE: ${{ inputs.service }} TEST_DATA: ${{ inputs.test-folder }} BASE_BACKUP: ${{ inputs.base-backup }} run: | diff --git a/src/cmd/sanity_test/common/common.go b/src/cmd/sanity_test/common/common.go new file mode 100644 index 000000000..344d6dc19 --- /dev/null +++ b/src/cmd/sanity_test/common/common.go @@ -0,0 +1,6 @@ +package common + +type PermissionInfo struct { + EntityID string + Roles []string +} diff --git a/src/cmd/sanity_test/common/utils.go b/src/cmd/sanity_test/common/utils.go new file mode 100644 index 000000000..e14fa86c6 --- /dev/null +++ b/src/cmd/sanity_test/common/utils.go @@ -0,0 +1,82 @@ +package common + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/logger" +) + +func Assert( + ctx context.Context, + passes func() bool, + header string, + expect, current any, +) { + if passes() { + return + } + + header = "Error: " + header + expected := fmt.Sprintf("* Expected: %+v", expect) + got := fmt.Sprintf("* Current: %+v", current) + + logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) + + fmt.Println(header) + fmt.Println(expected) + fmt.Println(got) + + os.Exit(1) +} + +func Fatal(ctx context.Context, msg string, err error) { + logger.CtxErr(ctx, err).Error("test failure: " + msg) + fmt.Println(msg+": ", err) + os.Exit(1) +} + +func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { + t, err := dttm.ExtractTime(name) + if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { + Fatal(ctx, "extracting time from name: "+name, err) + } + + return t, !errors.Is(err, dttm.ErrNoTimeString) +} + +func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { + if hasTime { + if bound.Before(check) { + logger.Ctx(ctx). + With("boundary_time", bound, "check_time", check). + Info("skipping restore folder: not older than time bound") + + return false + } + } + + return true +} + +func FilterSlice(sl []string, remove string) []string { + r := []string{} + + for _, s := range sl { + if !strings.EqualFold(s, remove) { + r = append(r, s) + } + } + + return r +} + +func LogAndPrint(ctx context.Context, tmpl string, vs ...any) { + logger.Ctx(ctx).Infof(tmpl, vs...) + fmt.Printf(tmpl+"\n", vs...) +} diff --git a/src/cmd/sanity_test/export/onedrive.go b/src/cmd/sanity_test/export/onedrive.go new file mode 100644 index 000000000..3d5564bcc --- /dev/null +++ b/src/cmd/sanity_test/export/onedrive.go @@ -0,0 +1,88 @@ +package export + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func CheckOneDriveExport( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + userID, folderName, dataFolder string, +) { + drive, err := client. + Users(). + ByUserId(userID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + // map itemID -> item size + var ( + fileSizes = make(map[string]int64) + exportFileSizes = make(map[string]int64) + startTime = time.Now() + ) + + err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return clues.Stack(err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(folderName, path) + if err != nil { + return clues.Stack(err) + } + + exportFileSizes[relPath] = info.Size() + if startTime.After(info.ModTime()) { + startTime = info.ModTime() + } + + return nil + }) + if err != nil { + fmt.Println("Error walking the path:", err) + } + + _ = restore.PopulateDriveDetails( + ctx, + client, + ptr.Val(drive.GetId()), + folderName, + dataFolder, + fileSizes, + map[string][]common.PermissionInfo{}, + startTime) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := exportFileSizes[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} diff --git a/src/cmd/sanity_test/restore/exchange.go b/src/cmd/sanity_test/restore/exchange.go index e484550de..2dc65e6e1 100644 --- a/src/cmd/sanity_test/restore/exchange.go +++ b/src/cmd/sanity_test/restore/exchange.go @@ -12,7 +12,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/filters" ) @@ -35,7 +35,7 @@ func CheckEmailRestoration( for { result, err := builder.Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting mail folders", err) + common.Fatal(ctx, "getting mail folders", err) } values := result.GetValue() @@ -79,7 +79,7 @@ func CheckEmailRestoration( ChildFolders(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting restore folder child folders", err) + common.Fatal(ctx, "getting restore folder child folders", err) } for _, fld := range childFolder.GetValue() { @@ -102,7 +102,7 @@ func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[ for fldName, expected := range messageCount { got := restoreMessageCount[fldName] - utils.Assert( + common.Assert( ctx, func() bool { return expected == got }, fmt.Sprintf("Restore item counts do not match: %s", fldName), @@ -142,7 +142,7 @@ func getAllMailSubFolders( ChildFolders(). Get(ctx, options) if err != nil { - utils.Fatal(ctx, "getting mail subfolders", err) + common.Fatal(ctx, "getting mail subfolders", err) } for _, child := range childFolder.GetValue() { @@ -194,7 +194,7 @@ func checkAllSubFolder( ChildFolders(). Get(ctx, options) if err != nil { - utils.Fatal(ctx, "getting mail subfolders", err) + common.Fatal(ctx, "getting mail subfolders", err) } for _, child := range childFolder.GetValue() { diff --git a/src/cmd/sanity_test/restore/onedrive.go b/src/cmd/sanity_test/restore/onedrive.go index ca332d608..87ea424c3 100644 --- a/src/cmd/sanity_test/restore/onedrive.go +++ b/src/cmd/sanity_test/restore/onedrive.go @@ -10,7 +10,7 @@ import ( msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" "golang.org/x/exp/slices" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/path" ) @@ -19,11 +19,6 @@ const ( owner = "owner" ) -type permissionInfo struct { - entityID string - roles []string -} - func CheckOneDriveRestoration( ctx context.Context, client *msgraphsdk.GraphServiceClient, @@ -36,7 +31,7 @@ func CheckOneDriveRestoration( Drive(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting the drive:", err) + common.Fatal(ctx, "getting the drive:", err) } checkDriveRestoration( @@ -66,15 +61,58 @@ func checkDriveRestoration( // map itemID -> item size fileSizes = make(map[string]int64) // map itemID -> permission id -> []permission roles - folderPermissions = make(map[string][]permissionInfo) + folderPermissions = make(map[string][]common.PermissionInfo) restoreFile = make(map[string]int64) - restoredFolderPermissions = make(map[string][]permissionInfo) + restoredFolderPermissions = make(map[string][]common.PermissionInfo) ) - var restoreFolderID string - ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) + restoreFolderID := PopulateDriveDetails( + ctx, + client, + driveID, + folderName, + dataFolder, + fileSizes, + folderPermissions, + startTime) + + getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) + + checkRestoredDriveItemPermissions( + ctx, + service, + skipPermissionTest, + folderPermissions, + restoredFolderPermissions) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := restoreFile[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} + +func PopulateDriveDetails( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, folderName, dataFolder string, + fileSizes map[string]int64, + folderPermissions map[string][]common.PermissionInfo, + startTime time.Time, +) string { + var restoreFolderID string + response, err := client. Drives(). ByDriveId(driveID). @@ -83,7 +121,7 @@ func checkDriveRestoration( Children(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting drive by id", err) + common.Fatal(ctx, "getting drive by id", err) } for _, driveItem := range response.GetValue() { @@ -98,7 +136,7 @@ func checkDriveRestoration( } if itemName != dataFolder { - utils.LogAndPrint(ctx, "test data for folder: %s", dataFolder) + common.LogAndPrint(ctx, "test data for folder: %s", dataFolder) continue } @@ -114,7 +152,7 @@ func checkDriveRestoration( // currently we don't restore blank folders. // skip permission check for empty folders if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - utils.LogAndPrint(ctx, "skipped empty folder: %s", itemName) + common.LogAndPrint(ctx, "skipped empty folder: %s", itemName) continue } @@ -122,37 +160,15 @@ func checkDriveRestoration( getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) } - getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) - - checkRestoredDriveItemPermissions( - ctx, - service, - skipPermissionTest, - folderPermissions, - restoredFolderPermissions) - - for fileName, expected := range fileSizes { - utils.LogAndPrint(ctx, "checking for file: %s", fileName) - - got := restoreFile[fileName] - - utils.Assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("different file size: %s", fileName), - expected, - got) - } - - fmt.Println("Success") + return restoreFolderID } func checkRestoredDriveItemPermissions( ctx context.Context, service path.ServiceType, skip bool, - folderPermissions map[string][]permissionInfo, - restoredFolderPermissions map[string][]permissionInfo, + folderPermissions map[string][]common.PermissionInfo, + restoredFolderPermissions map[string][]common.PermissionInfo, ) { if skip { return @@ -164,12 +180,12 @@ func checkRestoredDriveItemPermissions( **/ for folderName, permissions := range folderPermissions { - utils.LogAndPrint(ctx, "checking for folder: %s", folderName) + common.LogAndPrint(ctx, "checking for folder: %s", folderName) restoreFolderPerm := restoredFolderPermissions[folderName] if len(permissions) < 1 { - utils.LogAndPrint(ctx, "no permissions found in: %s", folderName) + common.LogAndPrint(ctx, "no permissions found in: %s", folderName) continue } @@ -179,7 +195,7 @@ func checkRestoredDriveItemPermissions( permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } } - utils.Assert( + common.Assert( ctx, permCheck, fmt.Sprintf("wrong number of restored permissions: %s", folderName), @@ -187,25 +203,25 @@ func checkRestoredDriveItemPermissions( restoreFolderPerm) for _, perm := range permissions { - eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } + eqID := func(pi common.PermissionInfo) bool { return strings.EqualFold(pi.EntityID, perm.EntityID) } i := slices.IndexFunc(restoreFolderPerm, eqID) - utils.Assert( + common.Assert( ctx, func() bool { return i >= 0 }, fmt.Sprintf("permission was restored in: %s", folderName), - perm.entityID, + perm.EntityID, restoreFolderPerm) // permissions should be sorted, so a by-index comparison works restored := restoreFolderPerm[i] - utils.Assert( + common.Assert( ctx, - func() bool { return slices.Equal(perm.roles, restored.roles) }, + func() bool { return slices.Equal(perm.Roles, restored.Roles) }, fmt.Sprintf("different roles restored: %s", folderName), - perm.roles, - restored.roles) + perm.Roles, + restored.Roles) } } } @@ -215,12 +231,12 @@ func getOneDriveChildFolder( client *msgraphsdk.GraphServiceClient, driveID, itemID, parentName string, fileSizes map[string]int64, - folderPermission map[string][]permissionInfo, + folderPermission map[string][]common.PermissionInfo, startTime time.Time, ) { response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting child folder", err) + common.Fatal(ctx, "getting child folder", err) } for _, driveItem := range response.GetValue() { @@ -230,8 +246,8 @@ func getOneDriveChildFolder( fullName = parentName + "/" + itemName ) - folderTime, hasTime := utils.MustGetTimeFromName(ctx, itemName) - if !utils.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { + folderTime, hasTime := common.MustGetTimeFromName(ctx, itemName) + if !common.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { continue } @@ -247,7 +263,7 @@ func getOneDriveChildFolder( // currently we don't restore blank folders. // skip permission check for empty folders if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - utils.LogAndPrint(ctx, "skipped empty folder: %s", fullName) + common.LogAndPrint(ctx, "skipped empty folder: %s", fullName) continue } @@ -262,7 +278,7 @@ func getRestoredDrive( client *msgraphsdk.GraphServiceClient, driveID, restoreFolderID string, restoreFile map[string]int64, - restoreFolder map[string][]permissionInfo, + restoreFolder map[string][]common.PermissionInfo, startTime time.Time, ) { restored, err := client. @@ -273,7 +289,7 @@ func getRestoredDrive( Children(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting child folder", err) + common.Fatal(ctx, "getting child folder", err) } for _, item := range restored.GetValue() { @@ -305,8 +321,8 @@ func permissionIn( ctx context.Context, client *msgraphsdk.GraphServiceClient, driveID, itemID string, -) []permissionInfo { - pi := []permissionInfo{} +) []common.PermissionInfo { + pi := []common.PermissionInfo{} pcr, err := client. Drives(). @@ -316,7 +332,7 @@ func permissionIn( Permissions(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting permission", err) + common.Fatal(ctx, "getting permission", err) } for _, perm := range pcr.GetValue() { @@ -326,7 +342,7 @@ func permissionIn( var ( gv2 = perm.GetGrantedToV2() - permInfo = permissionInfo{} + permInfo = common.PermissionInfo{} entityID string ) @@ -337,14 +353,14 @@ func permissionIn( entityID = ptr.Val(gv2.GetGroup().GetId()) } - roles := utils.FilterSlice(perm.GetRoles(), owner) + roles := common.FilterSlice(perm.GetRoles(), owner) for _, role := range roles { - permInfo.entityID = entityID - permInfo.roles = append(permInfo.roles, role) + permInfo.EntityID = entityID + permInfo.Roles = append(permInfo.Roles, role) } if len(roles) > 0 { - slices.Sort(permInfo.roles) + slices.Sort(permInfo.Roles) pi = append(pi, permInfo) } } diff --git a/src/cmd/sanity_test/restore/sharepoint.go b/src/cmd/sanity_test/restore/sharepoint.go index d84e28f6c..a5146d7a4 100644 --- a/src/cmd/sanity_test/restore/sharepoint.go +++ b/src/cmd/sanity_test/restore/sharepoint.go @@ -6,7 +6,7 @@ import ( msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/path" ) @@ -23,7 +23,7 @@ func CheckSharePointRestoration( Drive(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting the drive:", err) + common.Fatal(ctx, "getting the drive:", err) } checkDriveRestoration( diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index a4d455edd..44680ab47 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -9,8 +9,9 @@ import ( "github.com/alcionai/clues" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/export" "github.com/alcionai/corso/src/cmd/sanity_test/restore" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/logger" @@ -35,16 +36,16 @@ func main() { os.Getenv("AZURE_CLIENT_ID"), os.Getenv("AZURE_CLIENT_SECRET")) if err != nil { - utils.Fatal(ctx, "creating adapter", err) + common.Fatal(ctx, "creating adapter", err) } var ( client = msgraphsdk.NewGraphServiceClient(adapter) testUser = tconfig.GetM365UserID(ctx) testSite = tconfig.GetM365SiteID(ctx) - testService = os.Getenv("SANITY_RESTORE_SERVICE") - folder = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER")) - startTime, _ = utils.MustGetTimeFromName(ctx, folder) + testKind = os.Getenv("SANITY_TEST_KIND") // restore or export (cli arg?) + testService = os.Getenv("SANITY_TEST_SERVICE") + folder = strings.TrimSpace(os.Getenv("SANITY_TEST_FOLDER")) dataFolder = os.Getenv("TEST_DATA") baseBackupFolder = os.Getenv("BASE_BACKUP") ) @@ -53,19 +54,33 @@ func main() { ctx, "resource_owner", testUser, "service", testService, - "sanity_restore_folder", folder, - "start_time", startTime.Format(time.RFC3339Nano)) + "sanity_restore_folder", folder) logger.Ctx(ctx).Info("starting sanity test check") - switch testService { - case "exchange": - restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) - case "onedrive": - restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) - case "sharepoint": - restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + switch testKind { + case "restore": + startTime, _ := common.MustGetTimeFromName(ctx, folder) + clues.Add(ctx, "sanity_restore_start_time", startTime.Format(time.RFC3339)) + + switch testService { + case "exchange": + restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) + case "onedrive": + restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) + case "sharepoint": + restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + default: + common.Fatal(ctx, "unknown service for restore sanity tests", nil) + } + case "export": + switch testService { + case "onedrive": + export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder) + default: + common.Fatal(ctx, "unknown service for export sanity tests", nil) + } default: - utils.Fatal(ctx, "unknown service for restore sanity tests", nil) + common.Fatal(ctx, "unknown test kind (expected restore or export)", nil) } } From 1e126bd2afc14582aa21c2bed389753d5ef78037 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 08:39:17 -0700 Subject: [PATCH 07/17] Allow setting retention parameters on repo init (#3903) Take in information about retention so that when the repo is initialized we can configure retention Not currently exposed by CLI layer **Requires changing the interface for repo init so SDK consumers will require changes** --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3519 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/backup/help_e2e_test.go | 8 +- src/cli/repo/s3.go | 9 +- src/cli/repo/s3_e2e_test.go | 8 +- src/cli/restore/exchange_e2e_test.go | 8 +- src/internal/kopia/conn.go | 30 ++++++- src/internal/kopia/conn_test.go | 82 +++++++++++++++++-- src/internal/kopia/model_store_test.go | 2 +- src/internal/operations/maintenance_test.go | 2 +- src/internal/operations/restore_test.go | 2 +- src/internal/operations/test/helper_test.go | 2 +- src/internal/streamstore/collectables_test.go | 3 +- .../loadtest/repository_load_test.go | 3 +- src/pkg/repository/repository.go | 12 +-- src/pkg/repository/repository_test.go | 60 +++++++++++--- .../repository/repository_unexported_test.go | 9 +- 15 files changed, 203 insertions(+), 37 deletions(-) diff --git a/src/cli/backup/help_e2e_test.go b/src/cli/backup/help_e2e_test.go index b7100d333..f6b4edd83 100644 --- a/src/cli/backup/help_e2e_test.go +++ b/src/cli/backup/help_e2e_test.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -47,7 +48,12 @@ func prepM365Test( vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force) ctx = config.SetViper(ctx, vpr) - repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions()) + repo, err := repository.Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) return acct, st, repo, vpr, recorder, cfgFP diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 58f04764c..b611e93e5 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/pkg/account" + rep "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -158,7 +159,13 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) } - r, err := repository.Initialize(ctx, cfg.Account, cfg.Storage, opt) + // TODO(ashmrtn): Wire to flags for retention during repo init. + r, err := repository.Initialize( + ctx, + cfg.Account, + cfg.Storage, + opt, + rep.Retention{}) if err != nil { if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) { return nil diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index 4c3af6e5f..4b4975a6a 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) @@ -200,7 +201,12 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() { ctx = config.SetViper(ctx, vpr) // init the repo first - _, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + _, err = repository.Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 56b0e7255..904025116 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" @@ -83,7 +84,12 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() { ) // init the repo first - suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{}) + suite.repo, err = repository.Initialize( + ctx, + suite.acct, + suite.st, + control.Options{}, + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) suite.backupOps = make(map[path.CategoryType]operations.BackupOperation) diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index e9d20918a..7eac9df5c 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -74,7 +74,11 @@ func NewConn(s storage.Storage) *conn { } } -func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { +func (w *conn) Initialize( + ctx context.Context, + opts repository.Options, + retentionOpts repository.Retention, +) error { bst, err := blobStoreByProvider(ctx, opts, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") @@ -86,8 +90,23 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { return clues.Stack(err).WithClues(ctx) } - // todo - issue #75: nil here should be a storage.NewRepoOptions() - if err = repo.Initialize(ctx, bst, nil, cfg.CorsoPassphrase); err != nil { + rOpts := retention.NewOpts() + if err := rOpts.Set(retentionOpts); err != nil { + return clues.Wrap(err, "setting retention configuration").WithClues(ctx) + } + + blobCfg, _, err := rOpts.AsConfigs(ctx) + if err != nil { + return clues.Stack(err) + } + + // Minimal config for retention if caller requested it. + kopiaOpts := repo.NewRepositoryOptions{ + RetentionMode: blobCfg.RetentionMode, + RetentionPeriod: blobCfg.RetentionPeriod, + } + + if err = repo.Initialize(ctx, bst, &kopiaOpts, cfg.CorsoPassphrase); err != nil { if errors.Is(err, repo.ErrAlreadyInitialized) { return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) } @@ -111,7 +130,10 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { return clues.Stack(err).WithClues(ctx) } - return nil + // Calling with all parameters here will set extend object locks for + // maintenance. Parameters for actual retention should have been set during + // initialization and won't be updated again. + return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil() } func (w *conn) Connect(ctx context.Context, opts repository.Options) error { diff --git a/src/internal/kopia/conn_test.go b/src/internal/kopia/conn_test.go index 899447e00..36a2bcdfd 100644 --- a/src/internal/kopia/conn_test.go +++ b/src/internal/kopia/conn_test.go @@ -7,12 +7,15 @@ import ( "time" "github.com/alcionai/clues" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -26,7 +29,7 @@ func openKopiaRepo( st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - if err := k.Initialize(ctx, repository.Options{}); err != nil { + if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { return nil, err } @@ -82,13 +85,13 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() { st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Initialize(ctx, repository.Options{}) + err = k.Initialize(ctx, repository.Options{}, repository.Retention{}) assert.Error(t, err, clues.ToCore(err)) assert.ErrorIs(t, err, ErrorRepoAlreadyExists) } @@ -103,7 +106,7 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { st.Provider = storage.ProviderUnknown k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) assert.Error(t, err, clues.ToCore(err)) } @@ -413,7 +416,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx, opts) + err := k.Initialize(ctx, opts, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kopiaOpts := k.ClientOptions() @@ -453,3 +456,72 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { err = k.Close(ctx) assert.NoError(t, err, clues.ToCore(err)) } + +// --------------- +// integration tests that require object locking to be enabled on the bucket. +// --------------- +type ConnRetentionIntegrationSuite struct { + tester.Suite +} + +func TestConnRetentionIntegrationSuite(t *testing.T) { + suite.Run(t, &ConnRetentionIntegrationSuite{ + Suite: tester.NewRetentionSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}, + ), + }) +} + +// Test that providing retention doesn't change anything but retention values +// from the default values that kopia uses. +func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + st1 := storeTD.NewPrefixedS3Storage(t) + + k1 := NewConn(st1) + err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err)) + + st2 := storeTD.NewPrefixedS3Storage(t) + + k2 := NewConn(st2) + err = k2.Initialize( + ctx, + repository.Options{}, + repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }) + require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err)) + + dr1, ok := k1.Repository.(repo.DirectRepository) + require.True(t, ok, "getting direct repo 1") + + dr2, ok := k2.Repository.(repo.DirectRepository) + require.True(t, ok, "getting direct repo 2") + + format1 := dr1.FormatManager().ScrubbedContentFormat() + format2 := dr2.FormatManager().ScrubbedContentFormat() + + assert.Equal(t, format1, format2) + + blobCfg1, err := dr1.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting blob config 1: %v", clues.ToCore(err)) + + blobCfg2, err := dr2.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting retention config 2: %v", clues.ToCore(err)) + + assert.NotEqual(t, blobCfg1, blobCfg2) + + // Check to make sure retention not enabled unexpectedly. + checkRetentionParams(t, ctx, k1, blob.RetentionMode(""), 0, assert.False) + + // Some checks to make sure retention was fully initialized as expected. + checkRetentionParams(t, ctx, k2, blob.Governance, time.Hour*48, assert.True) +} diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index bd7d651ca..aa929ea9d 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -808,7 +808,7 @@ func openConnAndModelStore( st := storeTD.NewPrefixedS3Storage(t) c := NewConn(st) - err := c.Initialize(ctx, repository.Options{}) + err := c.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/operations/maintenance_test.go b/src/internal/operations/maintenance_test.go index 4ec6a3ee9..6c822e1cd 100644 --- a/src/internal/operations/maintenance_test.go +++ b/src/internal/operations/maintenance_test.go @@ -40,7 +40,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { ctx, flush := tester.NewContext(t) defer flush() - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kw, err := kopia.NewWrapper(k) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 5b124ee64..c97812a63 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -243,7 +243,7 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { suite.acct = tconfig.NewM365Account(t) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index f1bf65261..c826b3e44 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -102,7 +102,7 @@ func prepNewTestBackupOp( k := kopia.NewConn(bod.st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/streamstore/collectables_test.go b/src/internal/streamstore/collectables_test.go index d8897c146..ccbeab7c2 100644 --- a/src/internal/streamstore/collectables_test.go +++ b/src/internal/streamstore/collectables_test.go @@ -44,7 +44,8 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() { st := storeTD.NewPrefixedS3Storage(t) k := kopia.NewConn(st) - require.NoError(t, k.Initialize(ctx, repository.Options{})) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, clues.ToCore(err)) suite.kcloser = func() { k.Close(ctx) } diff --git a/src/pkg/repository/loadtest/repository_load_test.go b/src/pkg/repository/loadtest/repository_load_test.go index c723955d1..718ac678d 100644 --- a/src/pkg/repository/loadtest/repository_load_test.go +++ b/src/pkg/repository/loadtest/repository_load_test.go @@ -21,6 +21,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -102,7 +103,7 @@ func initM365Repo(t *testing.T) ( FailureHandling: control.FailFast, } - repo, err := repository.Initialize(ctx, ac, st, opts) + repo, err := repository.Initialize(ctx, ac, st, opts, ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) return ctx, repo, ac, st diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 1417d872d..291ef5fd0 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -25,7 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - rep "github.com/alcionai/corso/src/pkg/control/repository" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" @@ -82,7 +82,7 @@ type Repository interface { ) (operations.ExportOperation, error) NewMaintenance( ctx context.Context, - mOpts rep.Maintenance, + mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) DeleteBackup(ctx context.Context, id string) error BackupGetter @@ -117,7 +117,8 @@ func (r repository) GetID() string { // - validate the m365 account & secrets // - connect to the m365 account to ensure communication capability // - validate the provider config & secrets -// - initialize the kopia repo with the provider +// - initialize the kopia repo with the provider and retention parameters +// - update maintenance retention parameters as needed // - store the configuration details // - connect to the provider // - return the connected repository @@ -126,6 +127,7 @@ func Initialize( acct account.Account, s storage.Storage, opts control.Options, + retentionOpts ctrlRepo.Retention, ) (repo Repository, err error) { ctx = clues.Add( ctx, @@ -140,7 +142,7 @@ func Initialize( }() kopiaRef := kopia.NewConn(s) - if err := kopiaRef.Initialize(ctx, opts.Repo); err != nil { + if err := kopiaRef.Initialize(ctx, opts.Repo, retentionOpts); err != nil { // replace common internal errors so that sdk users can check results with errors.Is() if errors.Is(err, kopia.ErrorRepoAlreadyExists) { return nil, clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) @@ -408,7 +410,7 @@ func (r repository) NewRestore( func (r repository) NewMaintenance( ctx context.Context, - mOpts rep.Maintenance, + mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) { return operations.NewMaintenanceOperation( ctx, diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 34c55d4e7..2ee90b5da 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -60,7 +60,12 @@ func (suite *RepositoryUnitSuite) TestInitialize() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Initialize(ctx, test.account, st, control.DefaultOptions()) + _, err = Initialize( + ctx, + test.account, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -137,7 +142,12 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { defer flush() st := test.storage(t) - r, err := Initialize(ctx, test.account, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + test.account, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) if err == nil { defer func() { err := r.Close(ctx) @@ -169,7 +179,7 @@ func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() { st.SessionName = "corso-repository-test" st.SessionDuration = roleDuration.String() - r, err := Initialize(ctx, account.Account{}, st, control.Options{}) + r, err := Initialize(ctx, account.Account{}, st, control.Options{}, ctrlRepo.Retention{}) require.NoError(suite.T(), err) defer func() { @@ -186,7 +196,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + repo, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) // now re-connect @@ -203,7 +218,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) oldID := r.GetID() @@ -228,7 +248,12 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -250,7 +275,12 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg) @@ -269,7 +299,12 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) @@ -286,7 +321,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + repo, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err) // now re-connect @@ -350,7 +390,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { ctx, flush := tester.NewContext(t) defer flush() - repo, err := Initialize(ctx, acct, st, test.opts()) + repo, err := Initialize(ctx, acct, st, test.opts(), ctrlRepo.Retention{}) require.NoError(t, err) r := repo.(*repository) diff --git a/src/pkg/repository/repository_unexported_test.go b/src/pkg/repository/repository_unexported_test.go index e284fad30..0e600157d 100644 --- a/src/pkg/repository/repository_unexported_test.go +++ b/src/pkg/repository/repository_unexported_test.go @@ -240,7 +240,7 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() { require.NotNil(t, k) - err = k.Initialize(ctx, rep.Options{}) + err = k.Initialize(ctx, rep.Options{}, rep.Retention{}) require.NoError(t, err, clues.ToCore(err)) err = k.Connect(ctx, rep.Options{}) @@ -291,8 +291,11 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { k = kopia.NewConn(s) ) - require.NoError(t, k.Initialize(ctx, rep.Options{})) - require.NoError(t, k.Connect(ctx, rep.Options{})) + err := k.Initialize(ctx, rep.Options{}, rep.Retention{}) + require.NoError(t, err, "initializing repo: %v", clues.ToCore(err)) + + err = k.Connect(ctx, rep.Options{}) + require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err)) defer k.Close(ctx) From b559cba4e70d5c612b151a195327057be2af36e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:09:51 +0000 Subject: [PATCH 08/17] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.309=20to=201.44.311=20in=20/src=20(#?= =?UTF-8?q?3926)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.309 to 1.44.311.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.311 (2023-07-28)

Service Client Updates

  • service/sqs: Updates service documentation
    • Documentation changes related to SQS APIs.

SDK Bugs

  • aws/session: Modify resolving sso credential logic to fix stack overflow bug while configuring shared config profile via env var.

Release v1.44.310 (2023-07-27)

Service Client Updates

  • service/autoscaling: Updates service API, documentation, paginators, and examples
    • This release updates validation for instance types used in the AllowedInstanceTypes and ExcludedInstanceTypes parameters of the InstanceRequirements property of a MixedInstancesPolicy.
  • service/ebs: Updates service API and documentation
  • service/ec2: Updates service API, documentation, and waiters
    • SDK and documentation updates for Amazon Elastic Block Store APIs
  • service/eks: Updates service API
  • service/sagemaker: Updates service API and documentation
    • Expose ProfilerConfig attribute in SageMaker Search API response.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.309&new-version=1.44.311)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 992c60f69..fa4e729e3 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.309 + github.com/aws/aws-sdk-go v1.44.311 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 0c1ea866b..1a2e8345b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.309 h1:IPJOFBzXekakxmEpDwd4RTKmmBR6LIAiXgNsM51bWbU= -github.com/aws/aws-sdk-go v1.44.309/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.311 h1:60i8hyVMOXqabKJQPCq4qKRBQ6hRafI/WOcDxGM+J7Q= +github.com/aws/aws-sdk-go v1.44.311/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From a836e877af340b4e3318849fd5f8eb0d9ccf876f Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 10:47:42 -0700 Subject: [PATCH 09/17] Pull backup deletion logic into function (#3924) Make a separate function to handle backup deletion. Makes repo connection management a little bit easier since a simple defer will work now. Will also help keep things organized as we expand these tests to do other things like use the point-in-time function to check the deleted backups are still accessible, especially since that will require opening another connection to the repo --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3799 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/longevity_test/longevity.go | 71 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index 33de060f8..ef17b7e32 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "github.com/alcionai/clues" "github.com/spf13/cobra" "github.com/alcionai/corso/src/cli/config" @@ -16,6 +17,51 @@ import ( "github.com/alcionai/corso/src/pkg/store" ) +// deleteBackups connects to the repository and deletes all backups for +// service that are at least deletionDays old. Returns the IDs of all backups +// that were deleted. +func deleteBackups( + ctx context.Context, + service path.ServiceType, + deletionDays int, +) ([]string, error) { + ctx = clues.Add(ctx, "cutoff_days", deletionDays) + + r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, nil) + if err != nil { + return nil, clues.Wrap(err, "connecting to account").WithClues(ctx) + } + + defer r.Close(ctx) + + backups, err := r.BackupsByTag(ctx, store.Service(service)) + if err != nil { + return nil, clues.Wrap(err, "listing backups").WithClues(ctx) + } + + var ( + deleted []string + cutoff = time.Now().Add(-time.Hour * 24 * time.Duration(deletionDays)) + ) + + for _, backup := range backups { + if backup.StartAndEndTime.CompletedAt.Before(cutoff) { + if err := r.DeleteBackup(ctx, backup.ID.String()); err != nil { + return nil, clues.Wrap( + err, + "deleting backup"). + With("backup_id", backup.ID). + WithClues(ctx) + } + + deleted = append(deleted, backup.ID.String()) + logAndPrint(ctx, "Deleted backup %s", backup.ID.String()) + } + } + + return deleted, nil +} + func main() { var ( service path.ServiceType @@ -39,31 +85,16 @@ func main() { fatal(cc.Context(), "unknown service", nil) } - r, _, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil) - if err != nil { - fatal(cc.Context(), "unable to connect account", err) - } - - defer r.Close(cc.Context()) - - backups, err := r.BackupsByTag(cc.Context(), store.Service(service)) - if err != nil { - fatal(cc.Context(), "unable to find backups", err) - } + ctx := clues.Add(cc.Context(), "service", service) days, err := strconv.Atoi(os.Getenv("DELETION_DAYS")) if err != nil { - fatal(cc.Context(), "invalid no of days provided", nil) + fatal(ctx, "invalid number of days provided", nil) } - for _, backup := range backups { - if backup.StartAndEndTime.CompletedAt.Before(time.Now().AddDate(0, 0, -days)) { - if err := r.DeleteBackup(cc.Context(), backup.ID.String()); err != nil { - fatal(cc.Context(), "deleting backup", err) - } - - logAndPrint(cc.Context(), "Deleted backup %s", backup.ID.String()) - } + _, err = deleteBackups(ctx, service, days) + if err != nil { + fatal(cc.Context(), "deleting backups", clues.Stack(err)) } } From b12bb50a4aff88dc2d3c9afbc10126805eacc416 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:27:54 -0700 Subject: [PATCH 10/17] Retention update op (#3865) Create new operation type to update retention parameters for immutable backups --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/operations/retention_config.go | 77 +++++++++++++++++++ .../operations/retention_config_test.go | 74 ++++++++++++++++++ src/pkg/repository/repository.go | 16 ++++ 3 files changed, 167 insertions(+) create mode 100644 src/internal/operations/retention_config.go create mode 100644 src/internal/operations/retention_config_test.go diff --git a/src/internal/operations/retention_config.go b/src/internal/operations/retention_config.go new file mode 100644 index 000000000..ca8504382 --- /dev/null +++ b/src/internal/operations/retention_config.go @@ -0,0 +1,77 @@ +package operations + +import ( + "context" + "time" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/crash" + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" +) + +// RetentionConfigOperation wraps an operation with restore-specific props. +type RetentionConfigOperation struct { + operation + Results RetentionConfigResults + rcOpts repository.Retention +} + +// RetentionConfigResults aggregate the details of the results of the operation. +type RetentionConfigResults struct { + stats.StartAndEndTime +} + +// NewRetentionConfigOperation constructs and validates an operation to change +// retention parameters. +func NewRetentionConfigOperation( + ctx context.Context, + opts control.Options, + kw *kopia.Wrapper, + rcOpts repository.Retention, + bus events.Eventer, +) (RetentionConfigOperation, error) { + op := RetentionConfigOperation{ + operation: newOperation(opts, bus, count.New(), kw, nil), + rcOpts: rcOpts, + } + + // Don't run validation because we don't populate the model store. + + return op, nil +} + +func (op *RetentionConfigOperation) Run(ctx context.Context) (err error) { + defer func() { + if crErr := crash.Recovery(ctx, recover(), "retention_config"); crErr != nil { + err = crErr + } + }() + + op.Results.StartedAt = time.Now() + + // TODO(ashmrtn): Send telemetry? + + return op.do(ctx) +} + +func (op *RetentionConfigOperation) do(ctx context.Context) error { + defer func() { + op.Results.CompletedAt = time.Now() + }() + + err := op.operation.kopia.SetRetentionParameters(ctx, op.rcOpts) + if err != nil { + op.Status = Failed + return clues.Wrap(err, "running retention config operation") + } + + op.Status = Completed + + return nil +} diff --git a/src/internal/operations/retention_config_test.go b/src/internal/operations/retention_config_test.go new file mode 100644 index 000000000..ce57cd879 --- /dev/null +++ b/src/internal/operations/retention_config_test.go @@ -0,0 +1,74 @@ +package operations + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" +) + +type RetentionConfigOpIntegrationSuite struct { + tester.Suite +} + +func TestRetentionConfigOpIntegrationSuite(t *testing.T) { + suite.Run(t, &RetentionConfigOpIntegrationSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *RetentionConfigOpIntegrationSuite) TestRepoRetentionConfig() { + var ( + t = suite.T() + // need to initialize the repository before we can test connecting to it. + st = storeTD.NewPrefixedS3Storage(t) + k = kopia.NewConn(st) + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, clues.ToCore(err)) + + kw, err := kopia.NewWrapper(k) + // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe + // to close here. + k.Close(ctx) + + require.NoError(t, err, clues.ToCore(err)) + + defer kw.Close(ctx) + + // Only set extend locks parameter as other retention options require a bucket + // with object locking enabled. There's more complete tests in the kopia + // package. + rco, err := NewRetentionConfigOperation( + ctx, + control.DefaultOptions(), + kw, + repository.Retention{ + Extend: ptr.To(true), + }, + evmock.NewBus()) + require.NoError(t, err, clues.ToCore(err)) + + err = rco.Run(ctx) + assert.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, Completed, rco.Status) + assert.NotZero(t, rco.Results.StartedAt) + assert.NotZero(t, rco.Results.CompletedAt) + assert.NotEqual(t, rco.Results.StartedAt, rco.Results.CompletedAt) +} diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 291ef5fd0..d4b0f6eaa 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -84,6 +84,10 @@ type Repository interface { ctx context.Context, mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) + NewRetentionConfig( + ctx context.Context, + rcOpts ctrlRepo.Retention, + ) (operations.RetentionConfigOperation, error) DeleteBackup(ctx context.Context, id string) error BackupGetter // ConnectToM365 establishes graph api connections @@ -420,6 +424,18 @@ func (r repository) NewMaintenance( r.Bus) } +func (r repository) NewRetentionConfig( + ctx context.Context, + rcOpts ctrlRepo.Retention, +) (operations.RetentionConfigOperation, error) { + return operations.NewRetentionConfigOperation( + ctx, + r.Opts, + r.dataLayer, + rcOpts, + r.Bus) +} + // Backup retrieves a backup by id. func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) { return getBackup(ctx, id, store.NewKopiaStore(r.modelStore)) From e0368ba45af72be2d97e7e82285f76bba84c0636 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:00:53 -0700 Subject: [PATCH 11/17] Track and print stats about merging (#3930) Track and print information about the number of directories that were moved, deleted, recursively moved or deleted, and that didn't move at all This can help shed light on what's happening and generally help with debugging issues that may crop up --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3929 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 52 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 8be75009f..ea5a55dbd 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -28,6 +28,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph/metadata" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -852,6 +853,7 @@ func traverseBaseDir( expectedDirPath *path.Builder, dir fs.Directory, roots map[string]*treeMap, + stats *count.Bus, ) error { ctx = clues.Add(ctx, "old_dir_path", oldDirPath, @@ -889,9 +891,27 @@ func traverseBaseDir( // This directory was deleted. if upb == nil { currentPath = nil + + stats.Inc(statDel) } else { - // This directory was moved/renamed and the new location is in upb. + // This directory was explicitly mentioned and the new (possibly + // unchanged) location is in upb. currentPath = upb.ToBuilder() + + if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else { + stats.Inc(statMove) + } + } + } else { + // Just stats tracking stuff. + if currentPath == nil { + stats.Inc(statRecursiveDel) + } else if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else { + stats.Inc(statRecursiveMove) } } @@ -920,7 +940,8 @@ func traverseBaseDir( oldDirPath, currentPath, dEntry, - roots) + roots, + stats) }) if err != nil { return clues.Wrap(err, "traversing base directory") @@ -993,6 +1014,21 @@ func logBaseInfo(ctx context.Context, m ManifestEntry) { "base_backup_id", mbID) } +const ( + // statNoMove denotes an directory that wasn't moved at all. + statNoMove = "directories_not_moved" + // statMove denotes an directory that was explicitly moved. + statMove = "directories_explicitly_moved" + // statRecursiveMove denotes an directory that moved because one or more or + // its ancestors moved and it wasn't explicitly mentioned. + statRecursiveMove = "directories_recursively_moved" + // statDel denotes a directory that was explicitly deleted. + statDel = "directories_explicitly_deleted" + // statRecursiveDel denotes a directory that was deleted because one or more + // of its ancestors was deleted and it wasn't explicitly mentioned. + statRecursiveDel = "directories_recursively_deleted" +) + func inflateBaseTree( ctx context.Context, loader snapshotLoader, @@ -1058,10 +1094,13 @@ func inflateBaseTree( // The prefix is the tenant/service/owner/category set, which remains // otherwise unchecked in tree inflation below this point. newSubtreePath := subtreePath.ToBuilder() + if p, ok := updatedPaths[subtreePath.String()]; ok { newSubtreePath = p.ToBuilder() } + stats := count.New() + if err = traverseBaseDir( ictx, 0, @@ -1070,9 +1109,18 @@ func inflateBaseTree( newSubtreePath.Dir(), subtreeDir, roots, + stats, ); err != nil { return clues.Wrap(err, "traversing base snapshot").WithClues(ictx) } + + logger.Ctx(ctx).Infow( + "merge subtree stats", + statNoMove, stats.Get(statNoMove), + statMove, stats.Get(statMove), + statRecursiveMove, stats.Get(statRecursiveMove), + statDel, stats.Get(statDel), + statRecursiveDel, stats.Get(statRecursiveDel)) } return nil From f672325193679feda6fff0ce30400164c05f1495 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:46:49 -0700 Subject: [PATCH 12/17] Update clues dependency (#3933) Pull in fix for double hashing some values ---- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/go.mod | 2 +- src/go.sum | 4 ++-- src/internal/common/pii/url.go | 4 ++++ src/internal/observe/observe.go | 4 ++-- src/pkg/control/restore.go | 4 ---- src/pkg/filters/filters.go | 2 +- src/pkg/path/elements.go | 4 ---- src/pkg/path/path.go | 4 ---- src/pkg/selectors/scopes.go | 3 --- src/pkg/selectors/selectors.go | 2 +- 10 files changed, 11 insertions(+), 22 deletions(-) diff --git a/src/go.mod b/src/go.mod index fa4e729e3..9b03cf95b 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.2023071323 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a + github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go v1.44.311 github.com/aws/aws-xray-sdk-go v1.8.1 diff --git a/src/go.sum b/src/go.sum index 1a2e8345b..74f71dadf 100644 --- a/src/go.sum +++ b/src/go.sum @@ -53,8 +53,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a h1:mtJyeK/FhArTn06M5Lfgxk/GWnu8yqCGNN1BY16vjaA= -github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= +github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 h1:husF7eAYw2HEzgjfAmNy+ZLzyztJV2SyoUngSUo829Y= +github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 h1:w50/aVU+zRP5lvE86TSSCCYrrEyuXOlJA06R5RdTS8Y= github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377/go.mod h1:WH725ws0BYpZpTkVh4uqFHHPiiJuirl1Cm73jv5RYyA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/src/internal/common/pii/url.go b/src/internal/common/pii/url.go index 339bee19f..de6e5ed8a 100644 --- a/src/internal/common/pii/url.go +++ b/src/internal/common/pii/url.go @@ -78,6 +78,10 @@ func (u SafeURL) Format(fs fmt.State, _ rune) { fmt.Fprint(fs, u.Conceal()) } +func (u SafeURL) PlainString() string { + return u.URL +} + // String complies with Stringer to ensure the Conceal() version // of the url is printed anytime it gets transformed to a string. func (u SafeURL) String() string { diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index c05297587..50994eacb 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -563,8 +563,8 @@ func (b bulletf) String() string { // observe progress bar. Logged values should only use // the fmt %v to ensure Concealers hide PII. func plainString(v any) string { - if ps, ok := v.(clues.PlainStringer); ok { - return ps.PlainString() + if c, ok := v.(clues.Concealer); ok { + return c.PlainString() } return fmt.Sprintf("%v", v) diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 74cf88093..cce3bea9b 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -107,10 +107,6 @@ var ( // interface compliance required for handling PII _ clues.Concealer = &RestoreConfig{} _ fmt.Stringer = &RestoreConfig{} - - // interface compliance for the observe package to display - // values without concealing PII. - _ clues.PlainStringer = &RestoreConfig{} ) func (rc RestoreConfig) marshal() string { diff --git a/src/pkg/filters/filters.go b/src/pkg/filters/filters.go index 40c12a630..c697203b7 100644 --- a/src/pkg/filters/filters.go +++ b/src/pkg/filters/filters.go @@ -501,7 +501,7 @@ func suffixed(target, input string) bool { // Printers and PII control // ---------------------------------------------------------------------------------------------------- -var _ clues.PlainConcealer = &Filter{} +var _ clues.Concealer = &Filter{} var safeFilterValues = map[string]struct{}{"*": {}} diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index d1ca932dc..5a0802d50 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -54,10 +54,6 @@ var ( // interface compliance required for handling PII _ clues.Concealer = &Elements{} _ fmt.Stringer = &Elements{} - - // interface compliance for the observe package to display - // values without concealing PII. - _ clues.PlainStringer = &Elements{} ) // Elements are a PII Concealer-compliant slice of elements within a path. diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index ca63bf8fe..b7cd38da0 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -120,10 +120,6 @@ type Path interface { // is appropriately hidden from logging, errors, and other outputs. clues.Concealer fmt.Stringer - - // In the rare case that the path needs to get printed as a plain string, - // without obscuring values for PII. - clues.PlainStringer } // interface compliance required for handling PII diff --git a/src/pkg/selectors/scopes.go b/src/pkg/selectors/scopes.go index 2a860892b..aebd0f156 100644 --- a/src/pkg/selectors/scopes.go +++ b/src/pkg/selectors/scopes.go @@ -153,9 +153,6 @@ type ( // Primarily to ensure that root- or mid-tier scopes (such as folders) // cascade 'Any' matching to more granular categories. setDefaults() - - // Scopes need to comply with PII printing controls. - clues.PlainConcealer } // scopeT is the generic type interface of a scoper. scopeT interface { diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 1e148e04b..936bc3a32 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -341,7 +341,7 @@ func selectorAsIface[T any](s Selector) (T, error) { // Stringers and Concealers // --------------------------------------------------------------------------- -var _ clues.PlainConcealer = &Selector{} +var _ clues.Concealer = &Selector{} type loggableSelector struct { Service service `json:"service,omitempty"` From f0425a0f9ded79bfc9382dee2a04dd43a77ccd03 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:26:38 -0700 Subject: [PATCH 13/17] Add test to lookup deleted backup by opening old version of repo (#3925) Make helper function that allows * opening the repo at a previous point in time * listing backups * searching the list for backups in a given set This will allow us to open the repo just prior to the point in time at which backups were deleted so we can ensure PITR is working right This PR does not enable this functionality in CI, it only adds the code for it --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3799 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/longevity_test/longevity.go | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index ef17b7e32..8a5c01537 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -9,11 +9,13 @@ import ( "github.com/alcionai/clues" "github.com/spf13/cobra" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/store" ) @@ -62,6 +64,65 @@ func deleteBackups( return deleted, nil } +// pitrListBackups connects to the repository at the given point in time and +// lists the backups for service. It then checks the list of backups contains +// the backups in backupIDs. +// +//nolint:unused +//lint:ignore U1000 Waiting for full support. +func pitrListBackups( + ctx context.Context, + service path.ServiceType, + pitr time.Time, + backupIDs []string, +) error { + if len(backupIDs) == 0 { + return nil + } + + ctx = clues.Add(ctx, "pitr_time", pitr, "search_backups", backupIDs) + + // TODO(ashmrtn): This may be moved into CLI layer at some point when we add + // flags for opening a repo at a point in time. + cfg, err := config.GetConfigRepoDetails(ctx, true, true, nil) + if err != nil { + return clues.Wrap(err, "getting config info") + } + + opts := utils.ControlWithConfig(cfg) + opts.Repo.ViewTimestamp = &pitr + + r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, cfg.RepoID, opts) + if err != nil { + return clues.Wrap(err, "connecting to repo").WithClues(ctx) + } + + defer r.Close(ctx) + + backups, err := r.BackupsByTag(ctx, store.Service(service)) + if err != nil { + return clues.Wrap(err, "listing backups").WithClues(ctx) + } + + bups := map[string]struct{}{} + + for _, backup := range backups { + bups[backup.ID.String()] = struct{}{} + } + + ctx = clues.Add(ctx, "found_backups", maps.Keys(bups)) + + for _, backupID := range backupIDs { + if _, ok := bups[backupID]; !ok { + return clues.New("looking for backup"). + With("search_backup_id", backupID). + WithClues(ctx) + } + } + + return nil +} + func main() { var ( service path.ServiceType From dfbca1540d875bb4e8fc96c3bb0ee38ac9276b57 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:52:41 -0700 Subject: [PATCH 14/17] Minor refactor for handling of LoggableDir (#3934) Reuse existing Elements code to conceal information. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/pkg/path/elements.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index 5a0802d50..838cea114 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -2,7 +2,6 @@ package path import ( "fmt" - "strings" "github.com/alcionai/clues" @@ -119,16 +118,8 @@ func (el Elements) Last() string { // LoggableDir takes in a path reference (of any structure) and conceals any // non-standard elements (ids, filenames, foldernames, etc). func LoggableDir(ref string) string { - r := ref - n := strings.TrimSuffix(r, string(PathSeparator)) - - for n != r { - r = n - n = strings.TrimSuffix(r, string(PathSeparator)) - } - - elems := Split(r) - elems = pii.ConcealElements(elems, piiSafePathElems) - - return join(elems) + // Can't directly use Builder since that could return an error. Instead split + // into elements and use that. + split := Split(TrimTrailingSlash(ref)) + return Elements(split).Conceal() } From 5b455bfc4a3c803a984eb75de4c2eb74c7a35cd5 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:00:20 -0700 Subject: [PATCH 15/17] Better handling of clues, error, and log data if it's PII (#3935) Add better handling for * hiding possibly sensitive data when logging * adding possibly sensitive data to context clues * adding context clues to returned errors --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 44 +++++++++++++++++++++-------------- src/internal/kopia/wrapper.go | 16 ++++++------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index ea5a55dbd..3e954c244 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -252,7 +252,9 @@ func (cp *corsoProgress) FinishedHashingFile(fname string, bs int64) { sl[i] = string(rdt) } - logger.Ctx(context.Background()).Debugw("finished hashing file", "path", sl[2:]) + logger.Ctx(cp.ctx).Debugw( + "finished hashing file", + "path", clues.Hide(path.Elements(sl[2:]))) atomic.AddInt64(&cp.totalBytes, bs) } @@ -442,12 +444,12 @@ func streamBaseEntries( ctx = clues.Add( ctx, - "current_item_path", curPath, - "longest_prefix", longest) + "current_directory_path", curPath, + "longest_prefix", path.LoggableDir(longest)) err := dir.IterateEntries(ctx, func(innerCtx context.Context, entry fs.Entry) error { if err := innerCtx.Err(); err != nil { - return err + return clues.Stack(err).WithClues(ctx) } // Don't walk subdirectories in this function. @@ -464,7 +466,9 @@ func streamBaseEntries( entName, err := decodeElement(entry.Name()) if err != nil { - return clues.Wrap(err, "decoding entry name: "+entry.Name()) + return clues.Wrap(err, "decoding entry name"). + WithClues(ctx). + With("entry_name", entry.Name()) } // This entry was marked as deleted by a service that can't tell us the @@ -476,7 +480,7 @@ func streamBaseEntries( // For now assuming that item IDs don't need escaping. itemPath, err := curPath.AppendItem(entName) if err != nil { - return clues.Wrap(err, "getting full item path for base entry") + return clues.Wrap(err, "getting full item path for base entry").WithClues(ctx) } // We need the previous path so we can find this item in the base snapshot's @@ -485,7 +489,7 @@ func streamBaseEntries( // to look for. prevItemPath, err := prevPath.AppendItem(entName) if err != nil { - return clues.Wrap(err, "getting previous full item path for base entry") + return clues.Wrap(err, "getting previous full item path for base entry").WithClues(ctx) } // Meta files aren't in backup details since it's the set of items the user @@ -509,13 +513,15 @@ func streamBaseEntries( } if err := ctr(ctx, entry); err != nil { - return clues.Wrap(err, "executing callback on item").With("item_path", itemPath) + return clues.Wrap(err, "executing callback on item"). + WithClues(ctx). + With("item_path", itemPath) } return nil }) if err != nil { - return clues.Wrap(err, "traversing items in base snapshot directory") + return clues.Wrap(err, "traversing items in base snapshot directory").WithClues(ctx) } return nil @@ -826,7 +832,9 @@ func inflateCollectionTree( } if node.collection != nil && node.collection.State() == data.NotMovedState { - return nil, nil, clues.New("conflicting states for collection").With("changed_path", p) + return nil, nil, clues.New("conflicting states for collection"). + WithClues(ctx). + With("changed_path", p) } } @@ -860,7 +868,7 @@ func traverseBaseDir( "expected_dir_path", expectedDirPath) if depth >= maxInflateTraversalDepth { - return clues.New("base snapshot tree too tall") + return clues.New("base snapshot tree too tall").WithClues(ctx) } // Wrapper base64 encodes all file and folder names to avoid issues with @@ -868,7 +876,9 @@ func traverseBaseDir( // from kopia we need to do the decoding here. dirName, err := decodeElement(dir.Name()) if err != nil { - return clues.Wrap(err, "decoding base directory name").With("dir_name", dir.Name()) + return clues.Wrap(err, "decoding base directory name"). + WithClues(ctx). + With("dir_name", clues.Hide(dir.Name())) } // Form the path this directory would be at if the hierarchy remained the same @@ -944,7 +954,7 @@ func traverseBaseDir( stats) }) if err != nil { - return clues.Wrap(err, "traversing base directory") + return clues.Wrap(err, "traversing base directory").WithClues(ctx) } // We only need to add this base directory to the tree we're building if it @@ -961,7 +971,7 @@ func traverseBaseDir( // in the if-block though as that is an optimization. node := getTreeNode(roots, currentPath.Elements()) if node == nil { - return clues.New("getting tree node") + return clues.New("getting tree node").WithClues(ctx) } // Now that we have the node we need to check if there is a collection @@ -976,12 +986,12 @@ func traverseBaseDir( curP, err := path.FromDataLayerPath(currentPath.String(), false) if err != nil { - return clues.New("converting current path to path.Path") + return clues.New("converting current path to path.Path").WithClues(ctx) } oldP, err := path.FromDataLayerPath(oldDirPath.String(), false) if err != nil { - return clues.New("converting old path to path.Path") + return clues.New("converting old path to path.Path").WithClues(ctx) } node.baseDir = dir @@ -1172,7 +1182,7 @@ func inflateDirTree( } if len(roots) > 1 { - return nil, clues.New("multiple root directories") + return nil, clues.New("multiple root directories").WithClues(ctx) } var res fs.Directory diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 3963b30f6..7b1feca44 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -324,7 +324,7 @@ func (w Wrapper) makeSnapshotWithRoot( // Telling kopia to always flush may hide other errors if it fails while // flushing the write session (hence logging above). if err != nil { - return nil, clues.Wrap(err, "kopia backup") + return nil, clues.Wrap(err, "kopia backup").WithClues(ctx) } res := manifestToStats(man, progress, bc) @@ -369,7 +369,7 @@ func getDir( encodeElements(dirPath.PopFront().Elements()...)) if err != nil { if isErrEntryNotFound(err) { - err = clues.Stack(data.ErrNotFound, err) + err = clues.Stack(data.ErrNotFound, err).WithClues(ctx) } return nil, clues.Wrap(err, "getting nested object handle").WithClues(ctx) @@ -487,7 +487,7 @@ func (w Wrapper) ProduceRestoreCollections( // load it here. snapshotRoot, err := w.getSnapshotRoot(ctx, snapshotID) if err != nil { - return nil, clues.Wrap(err, "loading snapshot root") + return nil, clues.Wrap(err, "loading snapshot root").WithClues(ctx) } var ( @@ -507,8 +507,8 @@ func (w Wrapper) ProduceRestoreCollections( // items from a single directory instance lower down. ictx := clues.Add( ctx, - "item_path", itemPaths.StoragePath.String(), - "restore_path", itemPaths.RestorePath.String()) + "item_path", itemPaths.StoragePath, + "restore_path", itemPaths.RestorePath) parentStoragePath, err := itemPaths.StoragePath.Dir() if err != nil { @@ -552,7 +552,7 @@ func (w Wrapper) ProduceRestoreCollections( // then load the items from the directory. res, err := loadDirsAndItems(ctx, snapshotRoot, bcounter, dirsToItems, errs) if err != nil { - return nil, clues.Wrap(err, "loading items") + return nil, clues.Wrap(err, "loading items").WithClues(ctx) } return res, el.Failure() @@ -610,12 +610,12 @@ func (w Wrapper) RepoMaintenance( ) error { kopiaSafety, err := translateSafety(opts.Safety) if err != nil { - return clues.Wrap(err, "identifying safety level") + return clues.Wrap(err, "identifying safety level").WithClues(ctx) } mode, err := translateMode(opts.Type) if err != nil { - return clues.Wrap(err, "identifying maintenance mode") + return clues.Wrap(err, "identifying maintenance mode").WithClues(ctx) } currentOwner := w.c.ClientOptions().UsernameAtHost() From 2596fb91041450494be82f78736bc2ef1f62e497 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:36:36 -0700 Subject: [PATCH 16/17] Track when we skip merging due to DoNotMerge (#3938) Add some more nuanced tracking that takes into account the DoNotMerge flag and the New state of collections. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3929 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 40 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 3e954c244..9df1327f9 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -897,6 +897,8 @@ func traverseBaseDir( currentPath = currentPath.Append(dirName) } + var explicitMention bool + if upb, ok := updatedPaths[oldDirPath.String()]; ok { // This directory was deleted. if upb == nil { @@ -908,21 +910,16 @@ func traverseBaseDir( // unchanged) location is in upb. currentPath = upb.ToBuilder() - if oldDirPath.String() == currentPath.String() { - stats.Inc(statNoMove) - } else { - stats.Inc(statMove) - } + // Below we check if the collection was marked as new or DoNotMerge which + // disables merging behavior. That means we can't directly update stats + // here else we'll miss delta token refreshes and whatnot. Instead note + // that we did see the path explicitly so it's not counted as a recursive + // operation. + explicitMention = true } - } else { + } else if currentPath == nil { // Just stats tracking stuff. - if currentPath == nil { - stats.Inc(statRecursiveDel) - } else if oldDirPath.String() == currentPath.String() { - stats.Inc(statNoMove) - } else { - stats.Inc(statRecursiveMove) - } + stats.Inc(statRecursiveDel) } ctx = clues.Add(ctx, "new_path", currentPath) @@ -981,9 +978,20 @@ func traverseBaseDir( // directories. The expected usecase for this is delta token expiry in M365. if node.collection != nil && (node.collection.DoNotMergeItems() || node.collection.State() == data.NewState) { + stats.Inc(statSkipMerge) + return nil } + // Just stats tracking stuff. + if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else if explicitMention { + stats.Inc(statMove) + } else { + stats.Inc(statRecursiveMove) + } + curP, err := path.FromDataLayerPath(currentPath.String(), false) if err != nil { return clues.New("converting current path to path.Path").WithClues(ctx) @@ -1037,6 +1045,9 @@ const ( // statRecursiveDel denotes a directory that was deleted because one or more // of its ancestors was deleted and it wasn't explicitly mentioned. statRecursiveDel = "directories_recursively_deleted" + // statSkipMerge denotes the number of directories that weren't merged because + // they were marked either DoNotMerge or New. + statSkipMerge = "directories_skipped_merging" ) func inflateBaseTree( @@ -1130,7 +1141,8 @@ func inflateBaseTree( statMove, stats.Get(statMove), statRecursiveMove, stats.Get(statRecursiveMove), statDel, stats.Get(statDel), - statRecursiveDel, stats.Get(statRecursiveDel)) + statRecursiveDel, stats.Get(statRecursiveDel), + statSkipMerge, stats.Get(statSkipMerge)) } return nil From 57511635729f798288e272685f3e5ef7b8f32cd8 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Tue, 1 Aug 2023 11:33:51 +0530 Subject: [PATCH 17/17] Sanity tests for SharePoint export (#3928) This extends the existing export sanity tests to also cover SharePoint exports. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * closes https://github.com/alcionai/corso/issues/3889 #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../actions/backup-restore-test/action.yml | 4 +- src/cmd/sanity_test/export/sharepoint.go | 88 +++++++++++++++++++ src/cmd/sanity_test/sanity_tests.go | 2 + 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 src/cmd/sanity_test/export/sharepoint.go diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 56213d3e7..95e41d1bb 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -89,7 +89,7 @@ runs: id: export shell: bash working-directory: src - if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + if: ${{ inputs.service == 'onedrive' || inputs.service == 'sharepoint' }} run: | set -euo pipefail CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log @@ -105,7 +105,7 @@ runs: - name: Check export ${{ inputs.service }} ${{ inputs.kind }} shell: bash working-directory: src - if: ${{ inputs.service == 'onedrive' }} + if: ${{ inputs.service == 'onedrive' || inputs.service == 'sharepoint' }} env: SANITY_TEST_KIND: export SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }} diff --git a/src/cmd/sanity_test/export/sharepoint.go b/src/cmd/sanity_test/export/sharepoint.go new file mode 100644 index 000000000..55ab8ed5c --- /dev/null +++ b/src/cmd/sanity_test/export/sharepoint.go @@ -0,0 +1,88 @@ +package export + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func CheckSharePointExport( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + siteID, folderName, dataFolder string, +) { + drive, err := client. + Sites(). + BySiteId(siteID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + // map itemID -> item size + var ( + fileSizes = make(map[string]int64) + exportFileSizes = make(map[string]int64) + startTime = time.Now() + ) + + err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return clues.Stack(err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(folderName, path) + if err != nil { + return clues.Stack(err) + } + + exportFileSizes[relPath] = info.Size() + if startTime.After(info.ModTime()) { + startTime = info.ModTime() + } + + return nil + }) + if err != nil { + fmt.Println("Error walking the path:", err) + } + + _ = restore.PopulateDriveDetails( + ctx, + client, + ptr.Val(drive.GetId()), + folderName, + dataFolder, + fileSizes, + map[string][]common.PermissionInfo{}, + startTime) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := exportFileSizes[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index 44680ab47..84bce47a0 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -77,6 +77,8 @@ func main() { switch testService { case "onedrive": export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder) + case "sharepoint": + export.CheckSharePointExport(ctx, client, testSite, folder, dataFolder) default: common.Fatal(ctx, "unknown service for export sanity tests", nil) }