diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 608934ab7..ae979918d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,6 +214,7 @@ jobs: CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} + CORSO_LOG_FILE: stderr LOG_GRAPH_REQUESTS: true run: | set -euo pipefail @@ -225,7 +226,7 @@ jobs: -p 1 \ ./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests - # Upload the original go test log as an artifact for later review. + # Upload the original go test output as an artifact for later review. - name: Upload test log if: failure() uses: actions/upload-artifact@v3 diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index c5c8faec1..ced4a80af 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -53,6 +53,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} CORSO_LOAD_TESTS: true + CORSO_LOG_FILE: stderr CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }} CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }} CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 547f6f5ba..adb626d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - msgraph-beta-sdk-go replaces msgraph-sdk-go for new features. This can lead to long build times. - Handle case where user's drive has not been initialized - Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163)) +- Guest and External users (for cloud accounts) and non-on-premise users (for systems that use on-prem AD syncs) are now excluded from backup and restore operations. + ## [v0.1.0] (alpha) - 2023-01-13 diff --git a/README.md b/README.md index cdb00ddf3..33341e4e7 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ services, possibly beyond M365, will expand based on the interest and needs of t # Getting Started -See the [Corso Documentation](https://corsobackup.io/docs/intro) for more information. +See the [Corso Quickstart](https://corsobackup.io/docs/quickstart/) on our docs page. # Building Corso diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 63a1f6393..ac6455522 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -61,7 +61,7 @@ const ( const ( exchangeServiceCommand = "exchange" - exchangeServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" + exchangeServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" exchangeServiceCommandDeleteUseSuffix = "--backup " exchangeServiceCommandDetailsUseSuffix = "--backup " ) @@ -115,7 +115,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar( &user, utils.UserFN, nil, - "Backup Exchange data by user ID; accepts '"+utils.Wildcard+"' to select all users") + "Backup Exchange data by a user's email; accepts '"+utils.Wildcard+"' to select all users") fs.StringSliceVar( &exchangeData, utils.DataFN, nil, @@ -274,7 +274,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { users, err := m365.UserPNs(ctx, acct) if err != nil { - return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users")) + return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 user(s)")) } var ( @@ -347,7 +347,7 @@ func exchangeBackupCreateSelectors(userIDs, data []string) *selectors.ExchangeBa func validateExchangeBackupCreateFlags(userIDs, data []string) error { if len(userIDs) == 0 { - return errors.New("--user requires one or more ids or the wildcard *") + return errors.New("--user requires one or more email addresses or the wildcard '*'") } for _, d := range data { diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index 3454a9b3b..60a055dce 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -29,7 +29,7 @@ import ( const ( oneDriveServiceCommand = "onedrive" - oneDriveServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" + oneDriveServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" oneDriveServiceCommandDeleteUseSuffix = "--backup " oneDriveServiceCommandDetailsUseSuffix = "--backup " ) @@ -85,7 +85,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar(&user, utils.UserFN, nil, - "Backup OneDrive data by user ID; accepts '"+utils.Wildcard+"' to select all users. (required)") + "Backup OneDrive data by user's email address; accepts '"+utils.Wildcard+"' to select all users. (required)") options.AddOperationFlags(c) case listCommand: diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index d5624380f..4eb62dca0 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -27,9 +27,12 @@ import ( // setup and globals // ------------------------------------------------------------------------------------------------ +// sharePoint bucket info from flags var ( libraryItems []string libraryPaths []string + pageFolders []string + page []string site []string weburl []string @@ -38,6 +41,7 @@ var ( const ( dataLibraries = "libraries" + dataPages = "pages" ) const ( @@ -89,11 +93,10 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { utils.WebURLFN, nil, "Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") - // TODO: implement fs.StringSliceVar( &sharepointData, utils.DataFN, nil, - "Select one or more types of data to backup: "+dataLibraries+".") + "Select one or more types of data to backup: "+dataLibraries+" or "+dataPages+".") options.AddOperationFlags(c) case listCommand: @@ -128,11 +131,22 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { fs.StringArrayVar(&site, utils.SiteFN, nil, - "Backup SharePoint data by site ID; accepts '"+utils.Wildcard+"' to select all sites.") + "Select backup details by site ID; accepts '"+utils.Wildcard+"' to select all sites.") fs.StringSliceVar(&weburl, utils.WebURLFN, nil, - "Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") + "Select backup data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") + + fs.StringSliceVar( + &pageFolders, + utils.PageFN, nil, + "Select backup data by site ID; accepts '"+utils.Wildcard+"' to select all sites.") + + fs.StringSliceVar( + &page, + utils.PageItemFN, nil, + "Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.", + ) // info flags @@ -179,7 +193,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return nil } - if err := validateSharePointBackupCreateFlags(site, weburl); err != nil { + if err := validateSharePointBackupCreateFlags(site, weburl, sharepointData); err != nil { return err } @@ -200,7 +214,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs")) } - sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, gc) + sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, sharepointData, gc) if err != nil { return Only(ctx, errors.Wrap(err, "Retrieving up sharepoint sites by ID and WebURL")) } @@ -250,7 +264,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return nil } -func validateSharePointBackupCreateFlags(sites, weburls []string) error { +func validateSharePointBackupCreateFlags(sites, weburls, data []string) error { if len(sites) == 0 && len(weburls) == 0 { return errors.New( "requires one or more --" + @@ -260,13 +274,21 @@ func validateSharePointBackupCreateFlags(sites, weburls []string) error { ) } + for _, d := range data { + if d != dataLibraries && d != dataPages { + return errors.New( + d + " is an unrecognized data type; either " + dataLibraries + "or " + dataPages, + ) + } + } + return nil } // TODO: users might specify a data type, this only supports AllData(). func sharePointBackupCreateSelectors( ctx context.Context, - sites, weburls []string, + sites, weburls, data []string, gc *connector.GraphConnector, ) (*selectors.SharePointBackup, error) { if len(sites) == 0 && len(weburls) == 0 { @@ -297,7 +319,20 @@ func sharePointBackupCreateSelectors( } sel := selectors.NewSharePointBackup(union) - sel.Include(sel.AllData()) + if len(data) == 0 { + sel.Include(sel.AllData()) + + return sel, nil + } + + for _, d := range data { + switch d { + case dataLibraries: + sel.Include(sel.Libraries(selectors.Any())) + case dataPages: + sel.Include(sel.Pages(selectors.Any())) + } + } return sel, nil } diff --git a/src/cli/backup/sharepoint_test.go b/src/cli/backup/sharepoint_test.go index d568d4cc6..89e40a9f3 100644 --- a/src/cli/backup/sharepoint_test.go +++ b/src/cli/backup/sharepoint_test.go @@ -98,12 +98,13 @@ func (suite *SharePointSuite) TestValidateSharePointBackupCreateFlags() { } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl)) + test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl, nil)) }) } } func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { + comboString := []string{"id_1", "id_2"} gc := &connector.GraphConnector{ Sites: map[string]string{ "url_1": "id_1", @@ -115,6 +116,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { name string site []string weburl []string + data []string expect []string expectScopesLen int }{ @@ -163,7 +165,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { name: "duplicate sites and urls", site: []string{"id_1", "id_2"}, weburl: []string{"url_1", "url_2"}, - expect: []string{"id_1", "id_2"}, + expect: comboString, expectScopesLen: 2, }, { @@ -175,18 +177,25 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { }, { name: "unnecessary url wildcard", - site: []string{"id_1", "id_2"}, + site: comboString, weburl: []string{"url_1", utils.Wildcard}, expect: selectors.Any(), expectScopesLen: 2, }, + { + name: "Pages", + site: comboString, + data: []string{dataPages}, + expect: comboString, + expectScopesLen: 1, + }, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { ctx, flush := tester.NewContext() defer flush() - sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, gc) + sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, test.data, gc) require.NoError(t, err) assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners()) diff --git a/src/cli/restore/exchange.go b/src/cli/restore/exchange.go index 6d67fa03e..d0801061f 100644 --- a/src/cli/restore/exchange.go +++ b/src/cli/restore/exchange.go @@ -63,7 +63,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar(&user, utils.UserFN, nil, - "Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.") + "Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.") // email flags fs.StringSliceVar(&email, diff --git a/src/cli/restore/onedrive.go b/src/cli/restore/onedrive.go index 0932461a3..526db414b 100644 --- a/src/cli/restore/onedrive.go +++ b/src/cli/restore/onedrive.go @@ -49,7 +49,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar(&user, utils.UserFN, nil, - "Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.") + "Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.") // onedrive hierarchy (path/name) flags diff --git a/src/go.mod b/src/go.mod index e1e7dac46..4b4a4abe6 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,18 +2,16 @@ module github.com/alcionai/corso/src go 1.19 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c - require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.183 + github.com/aws/aws-sdk-go v1.44.184 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/kopia/kopia v0.12.2-0.20221229232524-ba938cf58cc8 - github.com/microsoft/kiota-abstractions-go v0.16.0 - github.com/microsoft/kiota-authentication-azure-go v0.6.0 - github.com/microsoft/kiota-http-go v0.13.0 + github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb + github.com/microsoft/kiota-abstractions-go v0.15.2 + github.com/microsoft/kiota-authentication-azure-go v0.5.0 + github.com/microsoft/kiota-http-go v0.11.0 github.com/microsoft/kiota-serialization-json-go v0.7.2 github.com/microsoftgraph/msgraph-beta-sdk-go v0.53.0 github.com/microsoftgraph/msgraph-sdk-go-core v0.33.0 @@ -22,7 +20,7 @@ require ( github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.14.0 + github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 github.com/tidwall/pretty v1.2.1 github.com/tomlazar/table v0.1.2 @@ -40,19 +38,17 @@ require ( github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/microsoft/kiota-serialization-form-go v0.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.5 // indirect - github.com/spf13/afero v1.9.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.34.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( @@ -114,14 +110,14 @@ require ( go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/crypto v0.5.0 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect - google.golang.org/grpc v1.51.0 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/grpc v1.52.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/src/go.sum b/src/go.sum index 2d120f6cd..72d8128dc 100644 --- a/src/go.sum +++ b/src/go.sum @@ -52,8 +52,6 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c h1:uUcdEZ4sz7kRYVWB3K49MBHdICRyXCVAzd4ZiY3lvo0= -github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c/go.mod h1:yzJV11S6N6XMboXt7oCO6Jy2jJHPeSMtA+KOJ9Y1548= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -62,8 +60,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk= -github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo= +github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -241,6 +239,8 @@ github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747h github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kopia/htmluibuild v0.0.0-20220928042710-9fdd02afb1e7 h1:WP5VfIQL7AaYkO4zTNSCsVOawTzudbc4tvLojvg0RKc= +github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb h1:0jLaKLiloYvRNbuHHpnQkJ7STAgzQ4z6n+KPa6Kyg7I= +github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb/go.mod h1:dtCyMCsWulG82o9bDopvnny9DpOQe0PnSDczJLuhnWA= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -251,8 +251,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -303,10 +303,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= -github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -362,8 +360,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 h1:lQ3JvmcVO1/AMFbabvUSJ4YtJRpEAX9Qza73p5j03sw= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1/go.mod h1:4aKqcbhASNqjbrG0h9BmkzcWvPJGxbef4B+j0XfFrZo= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= @@ -372,8 +370,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -387,8 +385,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tg123/go-htpasswd v1.2.0 h1:UKp34m9H467/xklxUxU15wKRru7fwXoTojtxg25ITF0= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -450,8 +448,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -747,8 +745,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc= -google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -765,8 +763,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index 2ba2e59a4..7eaf1c517 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -8,8 +8,8 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/discovery" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/exchange" - "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/support" @@ -44,7 +44,7 @@ func (gc *GraphConnector) DataCollections( return nil, err } - serviceEnabled, err := checkServiceEnabled(ctx, gc.Service, path.ServiceType(sels.Service), sels.DiscreteOwner) + serviceEnabled, err := checkServiceEnabled(ctx, gc.Owners.Users(), path.ServiceType(sels.Service), sels.DiscreteOwner) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) erro func checkServiceEnabled( ctx context.Context, - gs graph.Servicer, + au api.Users, service path.ServiceType, resource string, ) (bool, error) { @@ -147,7 +147,7 @@ func checkServiceEnabled( return true, nil } - _, info, err := discovery.User(ctx, gs, resource) + _, info, err := discovery.User(ctx, au, resource) if err != nil { return false, err } diff --git a/src/internal/connector/discovery/api/api.go b/src/internal/connector/discovery/api/api.go new file mode 100644 index 000000000..20fc1b25d --- /dev/null +++ b/src/internal/connector/discovery/api/api.go @@ -0,0 +1,57 @@ +package api + +import ( + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/account" +) + +// --------------------------------------------------------------------------- +// interfaces +// --------------------------------------------------------------------------- + +// Client is used to fulfill the interface for discovery +// queries that are traditionally backed by GraphAPI. A +// struct is used in this case, instead of deferring to +// pure function wrappers, so that the boundary separates the +// granular implementation of the graphAPI and kiota away +// from the exchange package's broader intents. +type Client struct { + Credentials account.M365Config + + // The stable service is re-usable for any non-paged request. + // This allows us to maintain performance across async requests. + stable graph.Servicer +} + +// NewClient produces a new exchange api client. Must be used in +// place of creating an ad-hoc client struct. +func NewClient(creds account.M365Config) (Client, error) { + s, err := newService(creds) + if err != nil { + return Client{}, err + } + + return Client{creds, s}, nil +} + +// service generates a new service. Used for paged and other long-running +// requests instead of the client's stable service, so that in-flight state +// within the adapter doesn't get clobbered +func (c Client) service() (*graph.Service, error) { + return newService(c.Credentials) +} + +func newService(creds account.M365Config) (*graph.Service, error) { + adapter, err := graph.CreateAdapter( + creds.AzureTenantID, + creds.AzureClientID, + creds.AzureClientSecret, + ) + if err != nil { + return nil, errors.Wrap(err, "generating graph api service client") + } + + return graph.NewService(adapter), nil +} diff --git a/src/internal/connector/discovery/api/users.go b/src/internal/connector/discovery/api/users.go new file mode 100644 index 000000000..ff41ee06f --- /dev/null +++ b/src/internal/connector/discovery/api/users.go @@ -0,0 +1,166 @@ +package api + +import ( + "context" + + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Users() Users { + return Users{c} +} + +// Users is an interface-compliant provider of the client. +type Users struct { + Client +} + +// --------------------------------------------------------------------------- +// structs +// --------------------------------------------------------------------------- + +type UserInfo struct { + DiscoveredServices map[path.ServiceType]struct{} +} + +func newUserInfo() *UserInfo { + return &UserInfo{ + DiscoveredServices: map[path.ServiceType]struct{}{ + path.ExchangeService: {}, + path.OneDriveService: {}, + }, + } +} + +// --------------------------------------------------------------------------- +// methods +// --------------------------------------------------------------------------- + +const ( + userSelectID = "id" + userSelectPrincipalName = "userPrincipalName" + userSelectDisplayName = "displayName" +) + +// Filter out both guest users, and (for on-prem installations) non-synced users. +// The latter filter makes an assumption that no on-prem users are guests; this might +// require more fine-tuned controls in the future. +// https://stackoverflow.com/questions/64044266/error-message-unsupported-or-invalid-query-filter-clause-specified-for-property +// +//nolint:lll +var userFilterNoGuests = "onPremisesSyncEnabled eq true OR userType eq 'Member'" + +func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration { + return &users.UsersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.UsersRequestBuilderGetQueryParameters{ + Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName}, + Filter: fs, + }, + } +} + +// GetAll retrieves all users. +func (c Users) GetAll(ctx context.Context) ([]models.Userable, error) { + service, err := c.service() + if err != nil { + return nil, err + } + + resp, err := service.Client().Users().Get(ctx, userOptions(&userFilterNoGuests)) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "getting all users") + } + + iter, err := msgraphgocore.NewPageIterator( + resp, + service.Adapter(), + models.CreateUserCollectionResponseFromDiscriminatorValue) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "constructing user iterator") + } + + var ( + iterErrs error + us = make([]models.Userable, 0) + ) + + iterator := func(item any) bool { + u, err := validateUser(item) + if err != nil { + iterErrs = support.WrapAndAppend("validating user", err, iterErrs) + } else { + us = append(us, u) + } + + return true + } + + if err := iter.Iterate(ctx, iterator); err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "iterating all users") + } + + return us, iterErrs +} + +func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, error) { + user, err := c.stable.Client().UsersById(userID).Get(ctx, nil) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "getting user by id") + } + + return user, nil +} + +func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) { + // Assume all services are enabled + // then filter down to only services the user has enabled + userInfo := newUserInfo() + + // TODO: OneDrive + + _, err := c.stable.Client().UsersById(userID).MailFolders().Get(ctx, nil) + if err != nil { + if !graph.IsErrExchangeMailFolderNotFound(err) { + return nil, support.ConnectorStackErrorTraceWrap(err, "getting user's exchange mailfolders") + } + + delete(userInfo.DiscoveredServices, path.ExchangeService) + } + + return userInfo, nil +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// validateUser ensures the item is a Userable, and contains the necessary +// identifiers that we handle with all users. +// returns the item as a Userable model. +func validateUser(item any) (models.Userable, error) { + m, ok := item.(models.Userable) + if !ok { + return nil, errors.Errorf("expected Userable, got %T", item) + } + + if m.GetId() == nil { + return nil, errors.Errorf("missing ID") + } + + if m.GetUserPrincipalName() == nil { + return nil, errors.New("missing principalName") + } + + return m, nil +} diff --git a/src/internal/connector/discovery/discovery_test.go b/src/internal/connector/discovery/api/users_test.go similarity index 84% rename from src/internal/connector/discovery/discovery_test.go rename to src/internal/connector/discovery/api/users_test.go index 15e0f3210..d13c27fbf 100644 --- a/src/internal/connector/discovery/discovery_test.go +++ b/src/internal/connector/discovery/api/users_test.go @@ -1,4 +1,4 @@ -package discovery +package api import ( "reflect" @@ -8,15 +8,15 @@ import ( "github.com/stretchr/testify/suite" ) -type DiscoverySuite struct { +type UsersUnitSuite struct { suite.Suite } -func TestDiscoverySuite(t *testing.T) { - suite.Run(t, new(DiscoverySuite)) +func TestUsersUnitSuite(t *testing.T) { + suite.Run(t, new(UsersUnitSuite)) } -func (suite *DiscoverySuite) TestParseUser() { +func (suite *UsersUnitSuite) TestValidateUser() { t := suite.T() name := "testuser" @@ -60,7 +60,7 @@ func (suite *DiscoverySuite) TestParseUser() { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseUser(tt.args) + got, err := validateUser(tt.args) if (err != nil) != tt.wantErr { t.Errorf("parseUser() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/src/internal/connector/discovery/discovery.go b/src/internal/connector/discovery/discovery.go index e9d0474db..3f75f74c4 100644 --- a/src/internal/connector/discovery/discovery.go +++ b/src/internal/connector/discovery/discovery.go @@ -3,125 +3,52 @@ package discovery import ( "context" - "github.com/microsoftgraph/msgraph-beta-sdk-go/models" - msuser "github.com/microsoftgraph/msgraph-beta-sdk-go/users" - msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" - "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/support" - "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/internal/connector/discovery/api" ) -const ( - userSelectID = "id" - userSelectPrincipalName = "userPrincipalName" - userSelectDisplayName = "displayName" -) +// --------------------------------------------------------------------------- +// interfaces +// --------------------------------------------------------------------------- -func Users(ctx context.Context, gs graph.Servicer, tenantID string) ([]models.Userable, error) { - users := make([]models.Userable, 0) - - options := &msuser.UsersRequestBuilderGetRequestConfiguration{ - QueryParameters: &msuser.UsersRequestBuilderGetQueryParameters{ - Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName}, - }, - } - - response, err := gs.Client().Users().Get(ctx, options) - if err != nil { - return nil, errors.Wrapf( - err, - "retrieving resources for tenant %s: %s", - tenantID, - support.ConnectorStackErrorTrace(err), - ) - } - - iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(), - models.CreateUserCollectionResponseFromDiscriminatorValue) - if err != nil { - return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) - } - - var iterErrs error - - callbackFunc := func(item interface{}) bool { - u, err := parseUser(item) - if err != nil { - iterErrs = support.WrapAndAppend("discovering users: ", err, iterErrs) - return true - } - - users = append(users, u) - - return true - } - - if err := iter.Iterate(ctx, callbackFunc); err != nil { - return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) - } - - return users, iterErrs +type getAller interface { + GetAll(context.Context) ([]models.Userable, error) } -type UserInfo struct { - DiscoveredServices map[path.ServiceType]struct{} +type getter interface { + GetByID(context.Context, string) (models.Userable, error) } -func User(ctx context.Context, gs graph.Servicer, userID string) (models.Userable, *UserInfo, error) { - user, err := gs.Client().UsersById(userID).Get(ctx, nil) +type getInfoer interface { + GetInfo(context.Context, string) (*api.UserInfo, error) +} + +type getWithInfoer interface { + getter + getInfoer +} + +// --------------------------------------------------------------------------- +// api +// --------------------------------------------------------------------------- + +// Users fetches all users in the tenant. +func Users(ctx context.Context, ga getAller) ([]models.Userable, error) { + return ga.GetAll(ctx) +} + +func User(ctx context.Context, gwi getWithInfoer, userID string) (models.Userable, *api.UserInfo, error) { + u, err := gwi.GetByID(ctx, userID) if err != nil { - return nil, nil, errors.Wrapf( - err, - "retrieving resource for tenant: %s", - support.ConnectorStackErrorTrace(err), - ) + return nil, nil, errors.Wrap(err, "getting user") } - // Assume all services are enabled - userInfo := &UserInfo{ - DiscoveredServices: map[path.ServiceType]struct{}{ - path.ExchangeService: {}, - path.OneDriveService: {}, - }, - } - - // Discover which services the user has enabled - - // Exchange: Query `MailFolders` - _, err = gs.Client().UsersById(userID).MailFolders().Get(ctx, nil) + ui, err := gwi.GetInfo(ctx, userID) if err != nil { - if !graph.IsErrExchangeMailFolderNotFound(err) { - return nil, nil, errors.Wrapf( - err, - "retrieving mail folders for tenant: %s", - support.ConnectorStackErrorTrace(err), - ) - } - - delete(userInfo.DiscoveredServices, path.ExchangeService) + return nil, nil, errors.Wrap(err, "getting user info") } - // TODO: OneDrive - - return user, userInfo, nil -} - -// parseUser extracts information from `models.Userable` we care about -func parseUser(item interface{}) (models.Userable, error) { - m, ok := item.(models.Userable) - if !ok { - return nil, errors.New("iteration retrieved non-User item") - } - - if m.GetId() == nil { - return nil, errors.Errorf("no ID for User") - } - - if m.GetUserPrincipalName() == nil { - return nil, errors.Errorf("no principal name for User: %s", *m.GetId()) - } - - return m, nil + return u, ui, nil } diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index e168ed082..ce37beab6 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -257,6 +257,24 @@ func (col *Collection) streamItems(ctx context.Context) { break } + // If the data is no longer available just return here and chalk it up + // as a success. There's no reason to retry and no way we can backup up + // enough information to restore the item anyway. + if e := graph.IsErrDeletedInFlight(err); e != nil { + atomic.AddInt64(&success, 1) + logger.Ctx(ctx).Infow( + "Graph reported item not found", + "error", + e, + "service", + path.ExchangeService.String(), + "category", + col.category.String, + ) + + return + } + if i < numberOfRetries { time.Sleep(time.Duration(3*(i+1)) * time.Second) } @@ -270,6 +288,16 @@ func (col *Collection) streamItems(ctx context.Context) { // attempted items. if e := graph.IsErrDeletedInFlight(err); e != nil { atomic.AddInt64(&success, 1) + logger.Ctx(ctx).Infow( + "Graph reported item not found", + "error", + e, + "service", + path.ExchangeService.String(), + "category", + col.category.String, + ) + return } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index fd0a8c4e2..c8ef5fc70 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -15,6 +15,7 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/connector/discovery" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" @@ -37,7 +38,9 @@ import ( // GraphRequestAdapter from the msgraph-sdk-go. Additional fields are for // bookkeeping and interfacing with other component. type GraphConnector struct { - Service graph.Servicer + Service graph.Servicer + Owners api.Client + tenant string Users map[string]string // key value Sites map[string]string // key value @@ -74,12 +77,15 @@ func NewGraphConnector(ctx context.Context, acct account.Account, r resource) (* credentials: m365, } - gService, err := gc.createService() + gc.Service, err = gc.createService() if err != nil { return nil, errors.Wrap(err, "creating service connection") } - gc.Service = gService + gc.Owners, err = api.NewClient(m365) + if err != nil { + return nil, errors.Wrap(err, "creating api client") + } // TODO(ashmrtn): When selectors only encapsulate a single resource owner that // is not a wildcard don't populate users or sites when making the connector. @@ -121,7 +127,7 @@ func (gc *GraphConnector) setTenantUsers(ctx context.Context) error { ctx, end := D.Span(ctx, "gc:setTenantUsers") defer end() - users, err := discovery.Users(ctx, gc.Service, gc.tenant) + users, err := discovery.Users(ctx, gc.Owners.Users()) if err != nil { return err } diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index d635ee6f9..71eff095a 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/suite" "golang.org/x/exp/maps" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/support" @@ -174,10 +175,10 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { ctx, flush := tester.NewContext() defer flush() - service, err := newConnector.createService() + owners, err := api.NewClient(suite.connector.credentials) require.NoError(suite.T(), err) - newConnector.Service = service + newConnector.Owners = owners suite.Empty(len(newConnector.Users)) err = newConnector.setTenantUsers(ctx) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 0cd261b24..f9485e16b 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -161,12 +161,16 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) { } // UpdateCollections initializes and adds the provided drive items to Collections -// A new collection is created for every drive folder (or package) +// A new collection is created for every drive folder (or package). +// oldPaths is the unchanged data that was loaded from the metadata file. +// newPaths starts as a copy of oldPaths and is updated as changes are found in +// the returned results. func (c *Collections) UpdateCollections( ctx context.Context, driveID, driveName string, items []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error { for _, item := range items { if item.GetRoot() != nil { @@ -197,23 +201,40 @@ func (c *Collections) UpdateCollections( switch { case item.GetFolder() != nil, item.GetPackage() != nil: - // Eventually, deletions of folders will be handled here so we may as well - // start off by saving the path.Path of the item instead of just the - // OneDrive parentRef or such. + if item.GetDeleted() != nil { + // Nested folders also return deleted delta results so we don't have to + // worry about doing a prefix search in the map to remove the subtree of + // the deleted folder/package. + delete(newPaths, *item.GetId()) + + // TODO(ashmrtn): Create a collection with state Deleted. + + break + } + + // Deletions of folders are handled in this case so we may as well start + // off by saving the path.Path of the item instead of just the OneDrive + // parentRef or such. folderPath, err := collectionPath.Append(*item.GetName(), false) if err != nil { logger.Ctx(ctx).Errorw("failed building collection path", "error", err) return err } - // TODO(ashmrtn): Handle deletions by removing this entry from the map. - // TODO(ashmrtn): Handle moves by setting the collection state if the - // collection doesn't already exist/have that state. - paths[*item.GetId()] = folderPath.String() + // Moved folders don't cause delta results for any subfolders nested in + // them. We need to go through and update paths to handle that. We only + // update newPaths so we don't accidentally clobber previous deletes. + // + // TODO(ashmrtn): Since we're also getting notifications about folder + // moves we may need to handle updates to a path of a collection we've + // already created and partially populated. + updatePath(newPaths, *item.GetId(), folderPath.String()) case item.GetFile() != nil: col, found := c.CollectionMap[collectionPath.String()] if !found { + // TODO(ashmrtn): Compare old and new path and set collection state + // accordingly. col = NewCollection( collectionPath, driveID, @@ -286,3 +307,27 @@ func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) boo return m.Matches(folderPathString) } + +func updatePath(paths map[string]string, id, newPath string) { + oldPath := paths[id] + if len(oldPath) == 0 { + paths[id] = newPath + return + } + + if oldPath == newPath { + return + } + + // We need to do a prefix search on the rest of the map to update the subtree. + // We don't need to make collections for all of these, as hierarchy merging in + // other components should take care of that. We do need to ensure that the + // resulting map contains all folders though so we know the next time around. + for folderID, p := range paths { + if !strings.HasPrefix(p, oldPath) { + continue + } + + paths[folderID] = strings.Replace(p, oldPath, newPath, 1) + } +} diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index b45d6f185..857c2dbda 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" @@ -96,6 +97,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { tests := []struct { testCase string items []models.DriveItemable + inputFolderMap map[string]string scope selectors.OneDriveScope expect assert.ErrorAssertionFunc expectedCollectionPaths []string @@ -109,6 +111,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("item", "item", testBaseDrivePath, false, false, false), }, + inputFolderMap: map[string]string{}, scope: anyFolder, expect: assert.Error, expectedMetadataPaths: map[string]string{}, @@ -118,8 +121,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("file", "file", testBaseDrivePath, true, false, false), }, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, @@ -137,6 +141,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("folder", "folder", testBaseDrivePath, false, true, false), }, + inputFolderMap: map[string]string{}, scope: anyFolder, expect: assert.NoError, expectedCollectionPaths: []string{}, @@ -154,6 +159,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("package", "package", testBaseDrivePath, false, false, true), }, + inputFolderMap: map[string]string{}, scope: anyFolder, expect: assert.NoError, expectedCollectionPaths: []string{}, @@ -175,8 +181,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, @@ -215,8 +222,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, - scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], + expect: assert.NoError, expectedCollectionPaths: append( expectedPathAsSlice( suite.T(), @@ -257,12 +265,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false), driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false), - driveItem("folder", "folder", testBaseDrivePath+folderSub, false, true, false), + driveItem("folder2", "folder", testBaseDrivePath+folderSub, false, true, false), driveItem("package", "package", testBaseDrivePath, false, false, true), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, + inputFolderMap: map[string]string{}, scope: (&selectors.OneDriveBackup{}). Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0], expect: assert.NoError, @@ -276,7 +285,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedFileCount: 1, expectedContainerCount: 1, expectedMetadataPaths: map[string]string{ - "folder": expectedPathAsSlice( + "folder2": expectedPathAsSlice( suite.T(), tenant, user, @@ -295,8 +304,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, - scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, @@ -309,6 +319,231 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { // No child folders for subfolder so nothing here. expectedMetadataPaths: map[string]string{}, }, + { + testCase: "not moved folder tree", + items: []models.DriveItemable{ + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + }, + { + testCase: "moved folder tree", + items: []models.DriveItemable{ + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + }, + { + testCase: "moved folder tree and subfolder 1", + items: []models.DriveItemable{ + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/subfolder", + )[0], + }, + }, + { + testCase: "moved folder tree and subfolder 2", + items: []models.DriveItemable{ + driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/subfolder", + )[0], + }, + }, + { + testCase: "deleted folder and package", + items: []models.DriveItemable{ + delItem("folder", testBaseDrivePath, false, true, false), + delItem("package", testBaseDrivePath, false, false, true), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "package": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/package", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{}, + }, + { + testCase: "delete folder tree move subfolder", + items: []models.DriveItemable{ + delItem("folder", testBaseDrivePath, false, true, false), + driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/subfolder", + )[0], + }, + }, } for _, tt := range tests { @@ -316,7 +551,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { ctx, flush := tester.NewContext() defer flush() - paths := map[string]string{} + outputFolderMap := map[string]string{} + maps.Copy(outputFolderMap, tt.inputFolderMap) c := NewCollections( tenant, user, @@ -326,7 +562,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", "General", tt.items, paths) + err := c.UpdateCollections(ctx, "driveID", "General", tt.items, tt.inputFolderMap, outputFolderMap) tt.expect(t, err) assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count") @@ -336,7 +572,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { assert.Contains(t, c.CollectionMap, collPath) } - assert.Equal(t, tt.expectedMetadataPaths, paths) + assert.Equal(t, tt.expectedMetadataPaths, outputFolderMap) }) } } @@ -361,3 +597,26 @@ func driveItem(id string, name string, path string, isFile, isFolder, isPackage return item } + +// delItem creates a DriveItemable that is marked as deleted. path must be set +// to the base drive path. +func delItem(id string, path string, isFile, isFolder, isPackage bool) models.DriveItemable { + item := models.NewDriveItem() + item.SetId(&id) + item.SetDeleted(models.NewDeleted()) + + parentReference := models.NewItemReference() + parentReference.SetPath(&path) + item.SetParentReference(parentReference) + + switch { + case isFile: + item.SetFile(models.NewFile()) + case isFolder: + item.SetFolder(models.NewFolder()) + case isPackage: + item.SetPackage(models.NewPackage_escaped()) + } + + return item +} diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 07da01521..937e739af 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -13,6 +13,7 @@ import ( "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/pkg/errors" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" @@ -167,7 +168,8 @@ type itemCollector func( ctx context.Context, driveID, driveName string, driveItems []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error // collectItems will enumerate all items in the specified drive and hand them to the @@ -182,9 +184,12 @@ func collectItems( newDeltaURL = "" // TODO(ashmrtn): Eventually this should probably be a parameter so we can // take in previous paths. - paths = map[string]string{} + oldPaths = map[string]string{} + newPaths = map[string]string{} ) + maps.Copy(newPaths, oldPaths) + // TODO: Specify a timestamp in the delta query // https://docs.microsoft.com/en-us/graph/api/driveitem-delta? // view=graph-rest-1.0&tabs=http#example-4-retrieving-delta-results-using-a-timestamp @@ -221,7 +226,7 @@ func collectItems( ) } - err = collector(ctx, driveID, driveName, r.GetValue(), paths) + err = collector(ctx, driveID, driveName, r.GetValue(), oldPaths, newPaths) if err != nil { return "", nil, err } @@ -240,7 +245,7 @@ func collectItems( builder = msdrives.NewItemRootDeltaRequestBuilder(*nextLink, service.Adapter()) } - return newDeltaURL, paths, nil + return newDeltaURL, newPaths, nil } // getFolder will lookup the specified folder name under `parentFolderID` @@ -356,7 +361,8 @@ func GetAllFolders( innerCtx context.Context, driveID, driveName string, items []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error { for _, item := range items { // Skip the root item. diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 7c94b0ea7..29a25b5c8 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -99,7 +99,8 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { ctx context.Context, driveID, driveName string, items []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error { for _, item := range items { if item.GetFile() != nil { diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index 2227eefda..62d628bf7 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -88,6 +88,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() { defer flush() paths := map[string]string{} + newPaths := map[string]string{} c := onedrive.NewCollections( tenant, site, @@ -96,7 +97,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() { &MockGraphService{}, nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths) + err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths, newPaths) test.expect(t, err) assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index d9320f2a6..52452ffa9 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -134,6 +134,7 @@ type corsoProgress struct { toMerge map[string]path.Path mu sync.RWMutex totalBytes int64 + errs *multierror.Error } // Kopia interface function used as a callback when kopia finishes processing a @@ -162,8 +163,13 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) { // These items were sourced from a base snapshot or were cached in kopia so we // never had to materialize their details in-memory. if d.info == nil { - // TODO(ashmrtn): We should probably be returning an error here? if d.prevPath == nil { + cp.errs = multierror.Append(cp.errs, errors.Errorf( + "item sourced from previous backup with no previous path. Service: %s, Category: %s", + d.repoPath.Service().String(), + d.repoPath.Category().String(), + )) + return } diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index a3a865cd8..57ce9fd56 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -468,7 +468,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { for k, v := range ci { if cachedTest.cached { - cp.CachedFile(k, 42) + cp.CachedFile(k, v.totalBytes) } cp.FinishedFile(k, v.err) @@ -489,6 +489,38 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { } } +func (suite *CorsoProgressUnitSuite) TestFinishedFileCachedNoPrevPathErrors() { + t := suite.T() + bd := &details.Builder{} + cachedItems := map[string]testInfo{ + suite.targetFileName: { + info: &itemDetails{info: nil, repoPath: suite.targetFilePath}, + err: nil, + totalBytes: 100, + }, + } + cp := corsoProgress{ + UploadProgress: &snapshotfs.NullUploadProgress{}, + deets: bd, + pending: map[string]*itemDetails{}, + } + + for k, v := range cachedItems { + cp.put(k, v.info) + } + + require.Len(t, cp.pending, len(cachedItems)) + + for k, v := range cachedItems { + cp.CachedFile(k, v.totalBytes) + cp.FinishedFile(k, v.err) + } + + assert.Empty(t, cp.pending) + assert.Empty(t, bd.Details().Entries) + assert.Error(t, cp.errs.ErrorOrNil()) +} + func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchyNewItem() { t := suite.T() // Order of folders in hierarchy from root to leaf (excluding the item). @@ -995,7 +1027,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() { virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), }, ), @@ -1301,7 +1333,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(inboxFileName1)[0], time.Time{}, - bytes.NewReader(inboxFileData1), + io.NopCloser(bytes.NewReader(inboxFileData1)), ), virtualfs.NewStaticDirectory( encodeElements(personalDir)[0], @@ -1309,12 +1341,12 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(personalFileName1)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), virtualfs.StreamingFileWithModTimeFromReader( encodeElements(personalFileName2)[0], time.Time{}, - bytes.NewReader(testFileData2), + io.NopCloser(bytes.NewReader(testFileData2)), ), }, ), @@ -1324,7 +1356,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(workFileName1)[0], time.Time{}, - bytes.NewReader(testFileData3), + io.NopCloser(bytes.NewReader(testFileData3)), ), }, ), @@ -1941,7 +1973,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), }, ), @@ -1951,7 +1983,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName2)[0], time.Time{}, - bytes.NewReader(testFileData2), + io.NopCloser(bytes.NewReader(testFileData2)), ), }, ), @@ -1966,7 +1998,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName3)[0], time.Time{}, - bytes.NewReader(testFileData3), + io.NopCloser(bytes.NewReader(testFileData3)), ), }, ), @@ -1976,7 +2008,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName4)[0], time.Time{}, - bytes.NewReader(testFileData4), + io.NopCloser(bytes.NewReader(testFileData4)), ), }, ), @@ -2123,7 +2155,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(inboxFileName1)[0], time.Time{}, - bytes.NewReader(inboxFileData1), + io.NopCloser(bytes.NewReader(inboxFileData1)), ), }, ), @@ -2138,7 +2170,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(contactsFileName1)[0], time.Time{}, - bytes.NewReader(contactsFileData1), + io.NopCloser(bytes.NewReader(contactsFileData1)), ), }, ), @@ -2196,7 +2228,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(eventsFileName1)[0], time.Time{}, - bytes.NewReader(eventsFileData1), + io.NopCloser(bytes.NewReader(eventsFileData1)), ), }, ), diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index feab0e687..8171c853a 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -160,10 +160,11 @@ func (w Wrapper) BackupCollections( progress, ) if err != nil { - return nil, nil, nil, err + combinedErrs := multierror.Append(nil, err, progress.errs) + return nil, nil, nil, combinedErrs.ErrorOrNil() } - return s, progress.deets, progress.toMerge, nil + return s, progress.deets, progress.toMerge, progress.errs.ErrorOrNil() } func (w Wrapper) makeSnapshotWithRoot( diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 89dbb340d..92a8b93c6 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -205,10 +205,21 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { return opStats.writeErr } + opStats.gc = gc.AwaitStatus() + + if opStats.gc.ErrorCount > 0 { + opStats.writeErr = multierror.Append(nil, opStats.writeErr, errors.Errorf( + "%v errors reported while fetching item data", + opStats.gc.ErrorCount, + )).ErrorOrNil() + + // Need to exit before we set started to true else we'll report no errors. + return opStats.writeErr + } + // should always be 1, since backups are 1:1 with resourceOwners. opStats.resourceCount = 1 opStats.started = true - opStats.gc = gc.AwaitStatus() return err } diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index bead584b5..f64febe2e 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -106,6 +106,11 @@ func PreloadLoggingFlags() (string, string) { return "info", dlf } + // if not specified, attempt to fall back to env declaration. + if len(logfile) == 0 { + logfile = os.Getenv("CORSO_LOG_FILE") + } + if logfile == "-" { logfile = "stdout" } diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 4a1ec407b..2d23211e4 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -25,7 +25,7 @@ func Users(ctx context.Context, m365Account account.Account) ([]*User, error) { return nil, errors.Wrap(err, "could not initialize M365 graph connection") } - users, err := discovery.Users(ctx, gc.Service, m365Account.ID()) + users, err := discovery.Users(ctx, gc.Owners.Users()) if err != nil { return nil, err } diff --git a/website/package-lock.json b/website/package-lock.json index ef0577fb2..c781b7458 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -26,7 +26,7 @@ "react-dom": "^17.0.2", "sass": "^1.57.1", "tw-elements": "^1.0.0-alpha13", - "wowjs": "^1.1.3" + "wow.js": "^1.2.2" }, "devDependencies": { "@docusaurus/module-type-aliases": "2.2.0", @@ -14543,13 +14543,11 @@ "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", "license": "MIT" }, - "node_modules/wowjs": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz", - "integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==", - "dependencies": { - "animate.css": "latest" - } + "node_modules/wow.js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz", + "integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A==", + "deprecated": "deprecated in favour of aos (Animate On Scroll)" }, "node_modules/wrap-ansi": { "version": "8.0.1", @@ -24231,13 +24229,10 @@ "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==" }, - "wowjs": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz", - "integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==", - "requires": { - "animate.css": "latest" - } + "wow.js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz", + "integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A==" }, "wrap-ansi": { "version": "8.0.1", diff --git a/website/package.json b/website/package.json index 6c27fe784..0cb897b46 100644 --- a/website/package.json +++ b/website/package.json @@ -32,7 +32,7 @@ "react-dom": "^17.0.2", "sass": "^1.57.1", "tw-elements": "^1.0.0-alpha13", - "wowjs": "^1.1.3" + "wow.js": "^1.2.2" }, "devDependencies": { "@docusaurus/module-type-aliases": "2.2.0", diff --git a/website/src/components/parts/KeyLoveFAQ.js b/website/src/components/parts/KeyLoveFAQ.js index c07c35aba..fb13be58d 100644 --- a/website/src/components/parts/KeyLoveFAQ.js +++ b/website/src/components/parts/KeyLoveFAQ.js @@ -5,12 +5,12 @@ export default function KeyLoveFAQ() { const jarallaxRef = useRef(null); useEffect(() => { if (typeof window !== "undefined") { - const WOW = require("wowjs"); + const WOW = require("wow.js"); const father = require("feather-icons"); const jarallax = require("jarallax"); require("tw-elements"); - new WOW.WOW({ + new WOW({ live: false, }).init(); father.replace();