Merge branch 'main' into beta-library-switch

.github/workflows/ci.yml

@@ -214,6 +214,7 @@ jobs:
           CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
           CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+          CORSO_LOG_FILE: stderr
           LOG_GRAPH_REQUESTS: true
         run: |
           set -euo pipefail
@@ -225,7 +226,7 @@ jobs:
           -p 1          \
           ./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests
 
-      # Upload the original go test log as an artifact for later review.
+      # Upload the original go test output as an artifact for later review.
       - name: Upload test log
         if: failure()
         uses: actions/upload-artifact@v3

.github/workflows/load_test.yml

@@ -53,6 +53,7 @@ jobs:
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           CORSO_LOAD_TESTS: true 
+          CORSO_LOG_FILE: stderr
           CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
           CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
           CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }}

CHANGELOG.md

@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - msgraph-beta-sdk-go replaces msgraph-sdk-go for new features. This can lead to long build times. 
 - Handle case where user's drive has not been initialized
 - Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163))
+- Guest and External users (for cloud accounts) and non-on-premise users (for systems that use on-prem AD syncs) are now excluded from backup and restore operations.
+
 
 ## [v0.1.0] (alpha) - 2023-01-13
 

README.md

@@ -20,7 +20,7 @@ services, possibly beyond M365, will expand based on the interest and needs of t
 
 # Getting Started
 
-See the [Corso Documentation](https://corsobackup.io/docs/intro) for more information.
+See the [Corso Quickstart](https://corsobackup.io/docs/quickstart/) on our docs page.
 
 # Building Corso
 

src/cli/backup/exchange.go

@@ -61,7 +61,7 @@ const (
 
 const (
 	exchangeServiceCommand                 = "exchange"
-	exchangeServiceCommandCreateUseSuffix  = "--user <userId or email> | '" + utils.Wildcard + "'"
+	exchangeServiceCommandCreateUseSuffix  = "--user <email> | '" + utils.Wildcard + "'"
 	exchangeServiceCommandDeleteUseSuffix  = "--backup <backupId>"
 	exchangeServiceCommandDetailsUseSuffix = "--backup <backupId>"
 )
@@ -115,7 +115,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
 		fs.StringSliceVar(
 			&user,
 			utils.UserFN, nil,
-			"Backup Exchange data by user ID; accepts '"+utils.Wildcard+"' to select all users")
+			"Backup Exchange data by a user's email; accepts '"+utils.Wildcard+"' to select all users")
 		fs.StringSliceVar(
 			&exchangeData,
 			utils.DataFN, nil,
@@ -274,7 +274,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
 
 	users, err := m365.UserPNs(ctx, acct)
 	if err != nil {
-		return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users"))
+		return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 user(s)"))
 	}
 
 	var (
@@ -347,7 +347,7 @@ func exchangeBackupCreateSelectors(userIDs, data []string) *selectors.ExchangeBa
 
 func validateExchangeBackupCreateFlags(userIDs, data []string) error {
 	if len(userIDs) == 0 {
-		return errors.New("--user requires one or more ids or the wildcard *")
+		return errors.New("--user requires one or more email addresses or the wildcard '*'")
 	}
 
 	for _, d := range data {

src/cli/backup/onedrive.go

@@ -29,7 +29,7 @@ import (
 
 const (
 	oneDriveServiceCommand                 = "onedrive"
-	oneDriveServiceCommandCreateUseSuffix  = "--user <userId or email> | '" + utils.Wildcard + "'"
+	oneDriveServiceCommandCreateUseSuffix  = "--user <email> | '" + utils.Wildcard + "'"
 	oneDriveServiceCommandDeleteUseSuffix  = "--backup <backupId>"
 	oneDriveServiceCommandDetailsUseSuffix = "--backup <backupId>"
 )
@@ -85,7 +85,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
 
 		fs.StringSliceVar(&user,
 			utils.UserFN, nil,
-			"Backup OneDrive data by user ID; accepts '"+utils.Wildcard+"' to select all users. (required)")
+			"Backup OneDrive data by user's email address; accepts '"+utils.Wildcard+"' to select all users. (required)")
 		options.AddOperationFlags(c)
 
 	case listCommand:

src/cli/backup/sharepoint.go

@@ -27,9 +27,12 @@ import (
 // setup and globals
 // ------------------------------------------------------------------------------------------------
 
+// sharePoint bucket info from flags
 var (
 	libraryItems []string
 	libraryPaths []string
+	pageFolders  []string
+	page         []string
 	site         []string
 	weburl       []string
 
@@ -38,6 +41,7 @@ var (
 
 const (
 	dataLibraries = "libraries"
+	dataPages     = "pages"
 )
 
 const (
@@ -89,11 +93,10 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
 			utils.WebURLFN, nil,
 			"Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.")
 
-		// TODO: implement
 		fs.StringSliceVar(
 			&sharepointData,
 			utils.DataFN, nil,
-			"Select one or more types of data to backup: "+dataLibraries+".")
+			"Select one or more types of data to backup: "+dataLibraries+" or "+dataPages+".")
 		options.AddOperationFlags(c)
 
 	case listCommand:
@@ -128,11 +131,22 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
 
 		fs.StringArrayVar(&site,
 			utils.SiteFN, nil,
-			"Backup SharePoint data by site ID; accepts '"+utils.Wildcard+"' to select all sites.")
+			"Select backup details by site ID; accepts '"+utils.Wildcard+"' to select all sites.")
 
 		fs.StringSliceVar(&weburl,
 			utils.WebURLFN, nil,
-			"Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.")
+			"Select backup data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.")
+
+		fs.StringSliceVar(
+			&pageFolders,
+			utils.PageFN, nil,
+			"Select backup data by site ID; accepts '"+utils.Wildcard+"' to select all sites.")
+
+		fs.StringSliceVar(
+			&page,
+			utils.PageItemFN, nil,
+			"Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.",
+		)
 
 		// info flags
 
@@ -179,7 +193,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	if err := validateSharePointBackupCreateFlags(site, weburl); err != nil {
+	if err := validateSharePointBackupCreateFlags(site, weburl, sharepointData); err != nil {
 		return err
 	}
 
@@ -200,7 +214,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
 		return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs"))
 	}
 
-	sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, gc)
+	sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, sharepointData, gc)
 	if err != nil {
 		return Only(ctx, errors.Wrap(err, "Retrieving up sharepoint sites by ID and WebURL"))
 	}
@@ -250,7 +264,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func validateSharePointBackupCreateFlags(sites, weburls []string) error {
+func validateSharePointBackupCreateFlags(sites, weburls, data []string) error {
 	if len(sites) == 0 && len(weburls) == 0 {
 		return errors.New(
 			"requires one or more --" +
@@ -260,13 +274,21 @@ func validateSharePointBackupCreateFlags(sites, weburls []string) error {
 		)
 	}
 
+	for _, d := range data {
+		if d != dataLibraries && d != dataPages {
+			return errors.New(
+				d + " is an unrecognized data type; either  " + dataLibraries + "or " + dataPages,
+			)
+		}
+	}
+
 	return nil
 }
 
 // TODO: users might specify a data type, this only supports AllData().
 func sharePointBackupCreateSelectors(
 	ctx context.Context,
-	sites, weburls []string,
+	sites, weburls, data []string,
 	gc *connector.GraphConnector,
 ) (*selectors.SharePointBackup, error) {
 	if len(sites) == 0 && len(weburls) == 0 {
@@ -297,7 +319,20 @@ func sharePointBackupCreateSelectors(
 	}
 
 	sel := selectors.NewSharePointBackup(union)
-	sel.Include(sel.AllData())
+	if len(data) == 0 {
+		sel.Include(sel.AllData())
+
+		return sel, nil
+	}
+
+	for _, d := range data {
+		switch d {
+		case dataLibraries:
+			sel.Include(sel.Libraries(selectors.Any()))
+		case dataPages:
+			sel.Include(sel.Pages(selectors.Any()))
+		}
+	}
 
 	return sel, nil
 }

src/cli/backup/sharepoint_test.go

@@ -98,12 +98,13 @@ func (suite *SharePointSuite) TestValidateSharePointBackupCreateFlags() {
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
-			test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl))
+			test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl, nil))
 		})
 	}
 }
 
 func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
+	comboString := []string{"id_1", "id_2"}
 	gc := &connector.GraphConnector{
 		Sites: map[string]string{
 			"url_1": "id_1",
@@ -115,6 +116,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
 		name            string
 		site            []string
 		weburl          []string
+		data            []string
 		expect          []string
 		expectScopesLen int
 	}{
@@ -163,7 +165,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
 			name:            "duplicate sites and urls",
 			site:            []string{"id_1", "id_2"},
 			weburl:          []string{"url_1", "url_2"},
-			expect:          []string{"id_1", "id_2"},
+			expect:          comboString,
 			expectScopesLen: 2,
 		},
 		{
@@ -175,18 +177,25 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
 		},
 		{
 			name:            "unnecessary url wildcard",
-			site:            []string{"id_1", "id_2"},
+			site:            comboString,
 			weburl:          []string{"url_1", utils.Wildcard},
 			expect:          selectors.Any(),
 			expectScopesLen: 2,
 		},
+		{
+			name:            "Pages",
+			site:            comboString,
+			data:            []string{dataPages},
+			expect:          comboString,
+			expectScopesLen: 1,
+		},
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
 			ctx, flush := tester.NewContext()
 			defer flush()
 
-			sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, gc)
+			sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, test.data, gc)
 			require.NoError(t, err)
 
 			assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners())

src/cli/restore/exchange.go

@@ -63,7 +63,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
 
 		fs.StringSliceVar(&user,
 			utils.UserFN, nil,
-			"Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.")
+			"Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.")
 
 		// email flags
 		fs.StringSliceVar(&email,

src/cli/restore/onedrive.go

@@ -49,7 +49,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
 
 		fs.StringSliceVar(&user,
 			utils.UserFN, nil,
-			"Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.")
+			"Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.")
 
 		// onedrive hierarchy (path/name) flags
 

src/go.mod

@@ -2,18 +2,16 @@ module github.com/alcionai/corso/src
 
 go 1.19
 
-replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c
-
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
-	github.com/aws/aws-sdk-go v1.44.183
+	github.com/aws/aws-sdk-go v1.44.184
 	github.com/aws/aws-xray-sdk-go v1.8.0
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/kopia/kopia v0.12.2-0.20221229232524-ba938cf58cc8
+	github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb
-	github.com/microsoft/kiota-abstractions-go v0.16.0
+	github.com/microsoft/kiota-abstractions-go v0.15.2
-	github.com/microsoft/kiota-authentication-azure-go v0.6.0
+	github.com/microsoft/kiota-authentication-azure-go v0.5.0
-	github.com/microsoft/kiota-http-go v0.13.0
+	github.com/microsoft/kiota-http-go v0.11.0
 	github.com/microsoft/kiota-serialization-json-go v0.7.2
 	github.com/microsoftgraph/msgraph-beta-sdk-go v0.53.0
 	github.com/microsoftgraph/msgraph-sdk-go-core v0.33.0
@@ -22,7 +20,7 @@ require (
 	github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.14.0
+	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.1
 	github.com/tidwall/pretty v1.2.1
 	github.com/tomlazar/table v0.1.2
@@ -40,19 +38,17 @@ require (
 	github.com/dnaeon/go-vcr v1.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/microsoft/kiota-serialization-form-go v0.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.34.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 require (
@@ -114,14 +110,14 @@ require (
 	go.opentelemetry.io/otel/trace v1.11.2 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.3.0 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/net v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect
-	google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect
+	google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
-	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/grpc v1.52.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

src/go.sum

@@ -52,8 +52,6 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
-github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c h1:uUcdEZ4sz7kRYVWB3K49MBHdICRyXCVAzd4ZiY3lvo0=
-github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c/go.mod h1:yzJV11S6N6XMboXt7oCO6Jy2jJHPeSMtA+KOJ9Y1548=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -62,8 +60,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk=
+github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo=
-github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY=
 github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -241,6 +239,8 @@ github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747h
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kopia/htmluibuild v0.0.0-20220928042710-9fdd02afb1e7 h1:WP5VfIQL7AaYkO4zTNSCsVOawTzudbc4tvLojvg0RKc=
+github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb h1:0jLaKLiloYvRNbuHHpnQkJ7STAgzQ4z6n+KPa6Kyg7I=
+github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb/go.mod h1:dtCyMCsWulG82o9bDopvnny9DpOQe0PnSDczJLuhnWA=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -251,8 +251,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
-github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
@@ -303,10 +303,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
 github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
-github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
-github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
-github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg=
-github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas=
 github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
 github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
@@ -362,8 +360,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 h1:lQ3JvmcVO1/AMFbabvUSJ4YtJRpEAX9Qza73p5j03sw=
 github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1/go.mod h1:4aKqcbhASNqjbrG0h9BmkzcWvPJGxbef4B+j0XfFrZo=
-github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw=
+github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
-github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
+github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
 github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
 github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
@@ -372,8 +370,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU=
+github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
-github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As=
+github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -387,8 +385,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
+github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
-github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
+github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/tg123/go-htpasswd v1.2.0 h1:UKp34m9H467/xklxUxU15wKRru7fwXoTojtxg25ITF0=
 github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
 github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -450,8 +448,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -747,8 +745,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc=
+google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY=
-google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0=
+google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -765,8 +763,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
+google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk=
-google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
+google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

src/internal/connector/data_collections.go

@@ -8,8 +8,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/src/internal/connector/discovery"
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
 	"github.com/alcionai/corso/src/internal/connector/exchange"
-	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/onedrive"
 	"github.com/alcionai/corso/src/internal/connector/sharepoint"
 	"github.com/alcionai/corso/src/internal/connector/support"
@@ -44,7 +44,7 @@ func (gc *GraphConnector) DataCollections(
 		return nil, err
 	}
 
-	serviceEnabled, err := checkServiceEnabled(ctx, gc.Service, path.ServiceType(sels.Service), sels.DiscreteOwner)
+	serviceEnabled, err := checkServiceEnabled(ctx, gc.Owners.Users(), path.ServiceType(sels.Service), sels.DiscreteOwner)
 	if err != nil {
 		return nil, err
 	}
@@ -138,7 +138,7 @@ func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) erro
 
 func checkServiceEnabled(
 	ctx context.Context,
-	gs graph.Servicer,
+	au api.Users,
 	service path.ServiceType,
 	resource string,
 ) (bool, error) {
@@ -147,7 +147,7 @@ func checkServiceEnabled(
 		return true, nil
 	}
 
-	_, info, err := discovery.User(ctx, gs, resource)
+	_, info, err := discovery.User(ctx, au, resource)
 	if err != nil {
 		return false, err
 	}

src/internal/connector/discovery/api/api.go

@@ -0,0 +1,57 @@
+package api
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/pkg/account"
+)
+
+// ---------------------------------------------------------------------------
+// interfaces
+// ---------------------------------------------------------------------------
+
+// Client is used to fulfill the interface for discovery
+// queries that are traditionally backed by GraphAPI.  A
+// struct is used in this case, instead of deferring to
+// pure function wrappers, so that the boundary separates the
+// granular implementation of the graphAPI and kiota away
+// from the exchange package's broader intents.
+type Client struct {
+	Credentials account.M365Config
+
+	// The stable service is re-usable for any non-paged request.
+	// This allows us to maintain performance across async requests.
+	stable graph.Servicer
+}
+
+// NewClient produces a new exchange api client.  Must be used in
+// place of creating an ad-hoc client struct.
+func NewClient(creds account.M365Config) (Client, error) {
+	s, err := newService(creds)
+	if err != nil {
+		return Client{}, err
+	}
+
+	return Client{creds, s}, nil
+}
+
+// service generates a new service.  Used for paged and other long-running
+// requests instead of the client's stable service, so that in-flight state
+// within the adapter doesn't get clobbered
+func (c Client) service() (*graph.Service, error) {
+	return newService(c.Credentials)
+}
+
+func newService(creds account.M365Config) (*graph.Service, error) {
+	adapter, err := graph.CreateAdapter(
+		creds.AzureTenantID,
+		creds.AzureClientID,
+		creds.AzureClientSecret,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "generating graph api service client")
+	}
+
+	return graph.NewService(adapter), nil
+}

src/internal/connector/discovery/api/users.go

@@ -0,0 +1,166 @@
+package api
+
+import (
+	"context"
+
+	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	"github.com/microsoftgraph/msgraph-sdk-go/users"
+	"github.com/pkg/errors"
+
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/internal/connector/support"
+	"github.com/alcionai/corso/src/pkg/path"
+)
+
+// ---------------------------------------------------------------------------
+// controller
+// ---------------------------------------------------------------------------
+
+func (c Client) Users() Users {
+	return Users{c}
+}
+
+// Users is an interface-compliant provider of the client.
+type Users struct {
+	Client
+}
+
+// ---------------------------------------------------------------------------
+// structs
+// ---------------------------------------------------------------------------
+
+type UserInfo struct {
+	DiscoveredServices map[path.ServiceType]struct{}
+}
+
+func newUserInfo() *UserInfo {
+	return &UserInfo{
+		DiscoveredServices: map[path.ServiceType]struct{}{
+			path.ExchangeService: {},
+			path.OneDriveService: {},
+		},
+	}
+}
+
+// ---------------------------------------------------------------------------
+// methods
+// ---------------------------------------------------------------------------
+
+const (
+	userSelectID            = "id"
+	userSelectPrincipalName = "userPrincipalName"
+	userSelectDisplayName   = "displayName"
+)
+
+// Filter out both guest users, and (for on-prem installations) non-synced users.
+// The latter filter makes an assumption that no on-prem users are guests; this might
+// require more fine-tuned controls in the future.
+// https://stackoverflow.com/questions/64044266/error-message-unsupported-or-invalid-query-filter-clause-specified-for-property
+//
+//nolint:lll
+var userFilterNoGuests = "onPremisesSyncEnabled eq true OR userType eq 'Member'"
+
+func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration {
+	return &users.UsersRequestBuilderGetRequestConfiguration{
+		QueryParameters: &users.UsersRequestBuilderGetQueryParameters{
+			Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName},
+			Filter: fs,
+		},
+	}
+}
+
+// GetAll retrieves all users.
+func (c Users) GetAll(ctx context.Context) ([]models.Userable, error) {
+	service, err := c.service()
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := service.Client().Users().Get(ctx, userOptions(&userFilterNoGuests))
+	if err != nil {
+		return nil, support.ConnectorStackErrorTraceWrap(err, "getting all users")
+	}
+
+	iter, err := msgraphgocore.NewPageIterator(
+		resp,
+		service.Adapter(),
+		models.CreateUserCollectionResponseFromDiscriminatorValue)
+	if err != nil {
+		return nil, support.ConnectorStackErrorTraceWrap(err, "constructing user iterator")
+	}
+
+	var (
+		iterErrs error
+		us       = make([]models.Userable, 0)
+	)
+
+	iterator := func(item any) bool {
+		u, err := validateUser(item)
+		if err != nil {
+			iterErrs = support.WrapAndAppend("validating user", err, iterErrs)
+		} else {
+			us = append(us, u)
+		}
+
+		return true
+	}
+
+	if err := iter.Iterate(ctx, iterator); err != nil {
+		return nil, support.ConnectorStackErrorTraceWrap(err, "iterating all users")
+	}
+
+	return us, iterErrs
+}
+
+func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, error) {
+	user, err := c.stable.Client().UsersById(userID).Get(ctx, nil)
+	if err != nil {
+		return nil, support.ConnectorStackErrorTraceWrap(err, "getting user by id")
+	}
+
+	return user, nil
+}
+
+func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) {
+	// Assume all services are enabled
+	// then filter down to only services the user has enabled
+	userInfo := newUserInfo()
+
+	// TODO: OneDrive
+
+	_, err := c.stable.Client().UsersById(userID).MailFolders().Get(ctx, nil)
+	if err != nil {
+		if !graph.IsErrExchangeMailFolderNotFound(err) {
+			return nil, support.ConnectorStackErrorTraceWrap(err, "getting user's exchange mailfolders")
+		}
+
+		delete(userInfo.DiscoveredServices, path.ExchangeService)
+	}
+
+	return userInfo, nil
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+// validateUser ensures the item is a Userable, and contains the necessary
+// identifiers that we handle with all users.
+// returns the item as a Userable model.
+func validateUser(item any) (models.Userable, error) {
+	m, ok := item.(models.Userable)
+	if !ok {
+		return nil, errors.Errorf("expected Userable, got %T", item)
+	}
+
+	if m.GetId() == nil {
+		return nil, errors.Errorf("missing ID")
+	}
+
+	if m.GetUserPrincipalName() == nil {
+		return nil, errors.New("missing principalName")
+	}
+
+	return m, nil
+}

src/internal/connector/discovery/api/users_test.go

@@ -1,4 +1,4 @@
-package discovery
+package api
 
 import (
 	"reflect"
@@ -8,15 +8,15 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
-type DiscoverySuite struct {
+type UsersUnitSuite struct {
 	suite.Suite
 }
 
-func TestDiscoverySuite(t *testing.T) {
+func TestUsersUnitSuite(t *testing.T) {
-	suite.Run(t, new(DiscoverySuite))
+	suite.Run(t, new(UsersUnitSuite))
 }
 
-func (suite *DiscoverySuite) TestParseUser() {
+func (suite *UsersUnitSuite) TestValidateUser() {
 	t := suite.T()
 
 	name := "testuser"
@@ -60,7 +60,7 @@ func (suite *DiscoverySuite) TestParseUser() {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseUser(tt.args)
+			got, err := validateUser(tt.args)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("parseUser() error = %v, wantErr %v", err, tt.wantErr)
 				return

src/internal/connector/discovery/discovery.go

@@ -3,125 +3,52 @@ package discovery
 import (
 	"context"
 
-	"github.com/microsoftgraph/msgraph-beta-sdk-go/models"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	msuser "github.com/microsoftgraph/msgraph-beta-sdk-go/users"
-	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
 	"github.com/pkg/errors"
 
-	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
-	"github.com/alcionai/corso/src/internal/connector/support"
-	"github.com/alcionai/corso/src/pkg/path"
 )
 
-const (
+// ---------------------------------------------------------------------------
-	userSelectID            = "id"
+// interfaces
-	userSelectPrincipalName = "userPrincipalName"
+// ---------------------------------------------------------------------------
-	userSelectDisplayName   = "displayName"
-)
 
-func Users(ctx context.Context, gs graph.Servicer, tenantID string) ([]models.Userable, error) {
+type getAller interface {
-	users := make([]models.Userable, 0)
+	GetAll(context.Context) ([]models.Userable, error)
-
-	options := &msuser.UsersRequestBuilderGetRequestConfiguration{
-		QueryParameters: &msuser.UsersRequestBuilderGetQueryParameters{
-			Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName},
-		},
-	}
-
-	response, err := gs.Client().Users().Get(ctx, options)
-	if err != nil {
-		return nil, errors.Wrapf(
-			err,
-			"retrieving resources for tenant %s: %s",
-			tenantID,
-			support.ConnectorStackErrorTrace(err),
-		)
-	}
-
-	iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(),
-		models.CreateUserCollectionResponseFromDiscriminatorValue)
-	if err != nil {
-		return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
-	}
-
-	var iterErrs error
-
-	callbackFunc := func(item interface{}) bool {
-		u, err := parseUser(item)
-		if err != nil {
-			iterErrs = support.WrapAndAppend("discovering users: ", err, iterErrs)
-			return true
-		}
-
-		users = append(users, u)
-
-		return true
-	}
-
-	if err := iter.Iterate(ctx, callbackFunc); err != nil {
-		return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
-	}
-
-	return users, iterErrs
 }
 
-type UserInfo struct {
+type getter interface {
-	DiscoveredServices map[path.ServiceType]struct{}
+	GetByID(context.Context, string) (models.Userable, error)
 }
 
-func User(ctx context.Context, gs graph.Servicer, userID string) (models.Userable, *UserInfo, error) {
+type getInfoer interface {
-	user, err := gs.Client().UsersById(userID).Get(ctx, nil)
+	GetInfo(context.Context, string) (*api.UserInfo, error)
+}
+
+type getWithInfoer interface {
+	getter
+	getInfoer
+}
+
+// ---------------------------------------------------------------------------
+// api
+// ---------------------------------------------------------------------------
+
+// Users fetches all users in the tenant.
+func Users(ctx context.Context, ga getAller) ([]models.Userable, error) {
+	return ga.GetAll(ctx)
+}
+
+func User(ctx context.Context, gwi getWithInfoer, userID string) (models.Userable, *api.UserInfo, error) {
+	u, err := gwi.GetByID(ctx, userID)
 	if err != nil {
-		return nil, nil, errors.Wrapf(
+		return nil, nil, errors.Wrap(err, "getting user")
-			err,
-			"retrieving resource for tenant: %s",
-			support.ConnectorStackErrorTrace(err),
-		)
 	}
 
-	// Assume all services are enabled
+	ui, err := gwi.GetInfo(ctx, userID)
-	userInfo := &UserInfo{
-		DiscoveredServices: map[path.ServiceType]struct{}{
-			path.ExchangeService: {},
-			path.OneDriveService: {},
-		},
-	}
-
-	// Discover which services the user has enabled
-
-	// Exchange: Query `MailFolders`
-	_, err = gs.Client().UsersById(userID).MailFolders().Get(ctx, nil)
 	if err != nil {
-		if !graph.IsErrExchangeMailFolderNotFound(err) {
+		return nil, nil, errors.Wrap(err, "getting user info")
-			return nil, nil, errors.Wrapf(
-				err,
-				"retrieving mail folders for tenant: %s",
-				support.ConnectorStackErrorTrace(err),
-			)
-		}
-
-		delete(userInfo.DiscoveredServices, path.ExchangeService)
 	}
 
-	// TODO: OneDrive
+	return u, ui, nil
-
-	return user, userInfo, nil
-}
-
-// parseUser extracts information from `models.Userable` we care about
-func parseUser(item interface{}) (models.Userable, error) {
-	m, ok := item.(models.Userable)
-	if !ok {
-		return nil, errors.New("iteration retrieved non-User item")
-	}
-
-	if m.GetId() == nil {
-		return nil, errors.Errorf("no ID for User")
-	}
-
-	if m.GetUserPrincipalName() == nil {
-		return nil, errors.Errorf("no principal name for User: %s", *m.GetId())
-	}
-
-	return m, nil
 }

src/internal/connector/exchange/exchange_data_collection.go

@@ -257,6 +257,24 @@ func (col *Collection) streamItems(ctx context.Context) {
 					break
 				}
 
+				// If the data is no longer available just return here and chalk it up
+				// as a success. There's no reason to retry and no way we can backup up
+				// enough information to restore the item anyway.
+				if e := graph.IsErrDeletedInFlight(err); e != nil {
+					atomic.AddInt64(&success, 1)
+					logger.Ctx(ctx).Infow(
+						"Graph reported item not found",
+						"error",
+						e,
+						"service",
+						path.ExchangeService.String(),
+						"category",
+						col.category.String,
+					)
+
+					return
+				}
+
 				if i < numberOfRetries {
 					time.Sleep(time.Duration(3*(i+1)) * time.Second)
 				}
@@ -270,6 +288,16 @@ func (col *Collection) streamItems(ctx context.Context) {
 				// attempted items.
 				if e := graph.IsErrDeletedInFlight(err); e != nil {
 					atomic.AddInt64(&success, 1)
+					logger.Ctx(ctx).Infow(
+						"Graph reported item not found",
+						"error",
+						e,
+						"service",
+						path.ExchangeService.String(),
+						"category",
+						col.category.String,
+					)
+
 					return
 				}
 

src/internal/connector/graph_connector.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/exp/maps"
 
 	"github.com/alcionai/corso/src/internal/connector/discovery"
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
 	"github.com/alcionai/corso/src/internal/connector/exchange"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/onedrive"
@@ -37,7 +38,9 @@ import (
 // GraphRequestAdapter from the msgraph-sdk-go. Additional fields are for
 // bookkeeping and interfacing with other component.
 type GraphConnector struct {
-	Service     graph.Servicer
+	Service graph.Servicer
+	Owners  api.Client
+
 	tenant      string
 	Users       map[string]string // key<email> value<id>
 	Sites       map[string]string // key<???> value<???>
@@ -74,12 +77,15 @@ func NewGraphConnector(ctx context.Context, acct account.Account, r resource) (*
 		credentials: m365,
 	}
 
-	gService, err := gc.createService()
+	gc.Service, err = gc.createService()
 	if err != nil {
 		return nil, errors.Wrap(err, "creating service connection")
 	}
 
-	gc.Service = gService
+	gc.Owners, err = api.NewClient(m365)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating api client")
+	}
 
 	// TODO(ashmrtn): When selectors only encapsulate a single resource owner that
 	// is not a wildcard don't populate users or sites when making the connector.
@@ -121,7 +127,7 @@ func (gc *GraphConnector) setTenantUsers(ctx context.Context) error {
 	ctx, end := D.Span(ctx, "gc:setTenantUsers")
 	defer end()
 
-	users, err := discovery.Users(ctx, gc.Service, gc.tenant)
+	users, err := discovery.Users(ctx, gc.Owners.Users())
 	if err != nil {
 		return err
 	}

src/internal/connector/graph_connector_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/stretchr/testify/suite"
 	"golang.org/x/exp/maps"
 
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/mockconnector"
 	"github.com/alcionai/corso/src/internal/connector/support"
@@ -174,10 +175,10 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	service, err := newConnector.createService()
+	owners, err := api.NewClient(suite.connector.credentials)
 	require.NoError(suite.T(), err)
 
-	newConnector.Service = service
+	newConnector.Owners = owners
 
 	suite.Empty(len(newConnector.Users))
 	err = newConnector.setTenantUsers(ctx)

src/internal/connector/onedrive/collections.go

@@ -161,12 +161,16 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
 }
 
 // UpdateCollections initializes and adds the provided drive items to Collections
-// A new collection is created for every drive folder (or package)
+// A new collection is created for every drive folder (or package).
+// oldPaths is the unchanged data that was loaded from the metadata file.
+// newPaths starts as a copy of oldPaths and is updated as changes are found in
+// the returned results.
 func (c *Collections) UpdateCollections(
 	ctx context.Context,
 	driveID, driveName string,
 	items []models.DriveItemable,
-	paths map[string]string,
+	oldPaths map[string]string,
+	newPaths map[string]string,
 ) error {
 	for _, item := range items {
 		if item.GetRoot() != nil {
@@ -197,23 +201,40 @@ func (c *Collections) UpdateCollections(
 
 		switch {
 		case item.GetFolder() != nil, item.GetPackage() != nil:
-			// Eventually, deletions of folders will be handled here so we may as well
+			if item.GetDeleted() != nil {
-			// start off by saving the path.Path of the item instead of just the
+				// Nested folders also return deleted delta results so we don't have to
-			// OneDrive parentRef or such.
+				// worry about doing a prefix search in the map to remove the subtree of
+				// the deleted folder/package.
+				delete(newPaths, *item.GetId())
+
+				// TODO(ashmrtn): Create a collection with state Deleted.
+
+				break
+			}
+
+			// Deletions of folders are handled in this case so we may as well start
+			// off by saving the path.Path of the item instead of just the OneDrive
+			// parentRef or such.
 			folderPath, err := collectionPath.Append(*item.GetName(), false)
 			if err != nil {
 				logger.Ctx(ctx).Errorw("failed building collection path", "error", err)
 				return err
 			}
 
-			// TODO(ashmrtn): Handle deletions by removing this entry from the map.
+			// Moved folders don't cause delta results for any subfolders nested in
-			// TODO(ashmrtn): Handle moves by setting the collection state if the
+			// them. We need to go through and update paths to handle that. We only
-			// collection doesn't already exist/have that state.
+			// update newPaths so we don't accidentally clobber previous deletes.
-			paths[*item.GetId()] = folderPath.String()
+			//
+			// TODO(ashmrtn): Since we're also getting notifications about folder
+			// moves we may need to handle updates to a path of a collection we've
+			// already created and partially populated.
+			updatePath(newPaths, *item.GetId(), folderPath.String())
 
 		case item.GetFile() != nil:
 			col, found := c.CollectionMap[collectionPath.String()]
 			if !found {
+				// TODO(ashmrtn): Compare old and new path and set collection state
+				// accordingly.
 				col = NewCollection(
 					collectionPath,
 					driveID,
@@ -286,3 +307,27 @@ func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) boo
 
 	return m.Matches(folderPathString)
 }
+
+func updatePath(paths map[string]string, id, newPath string) {
+	oldPath := paths[id]
+	if len(oldPath) == 0 {
+		paths[id] = newPath
+		return
+	}
+
+	if oldPath == newPath {
+		return
+	}
+
+	// We need to do a prefix search on the rest of the map to update the subtree.
+	// We don't need to make collections for all of these, as hierarchy merging in
+	// other components should take care of that. We do need to ensure that the
+	// resulting map contains all folders though so we know the next time around.
+	for folderID, p := range paths {
+		if !strings.HasPrefix(p, oldPath) {
+			continue
+		}
+
+		paths[folderID] = strings.Replace(p, oldPath, newPath, 1)
+	}
+}

src/internal/connector/onedrive/collections_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"golang.org/x/exp/maps"
 
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/pkg/control"
@@ -96,6 +97,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 	tests := []struct {
 		testCase                string
 		items                   []models.DriveItemable
+		inputFolderMap          map[string]string
 		scope                   selectors.OneDriveScope
 		expect                  assert.ErrorAssertionFunc
 		expectedCollectionPaths []string
@@ -109,6 +111,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			items: []models.DriveItemable{
 				driveItem("item", "item", testBaseDrivePath, false, false, false),
 			},
+			inputFolderMap:        map[string]string{},
 			scope:                 anyFolder,
 			expect:                assert.Error,
 			expectedMetadataPaths: map[string]string{},
@@ -118,8 +121,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			items: []models.DriveItemable{
 				driveItem("file", "file", testBaseDrivePath, true, false, false),
 			},
-			scope:  anyFolder,
+			inputFolderMap: map[string]string{},
-			expect: assert.NoError,
+			scope:          anyFolder,
+			expect:         assert.NoError,
 			expectedCollectionPaths: expectedPathAsSlice(
 				suite.T(),
 				tenant,
@@ -137,6 +141,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			items: []models.DriveItemable{
 				driveItem("folder", "folder", testBaseDrivePath, false, true, false),
 			},
+			inputFolderMap:          map[string]string{},
 			scope:                   anyFolder,
 			expect:                  assert.NoError,
 			expectedCollectionPaths: []string{},
@@ -154,6 +159,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			items: []models.DriveItemable{
 				driveItem("package", "package", testBaseDrivePath, false, false, true),
 			},
+			inputFolderMap:          map[string]string{},
 			scope:                   anyFolder,
 			expect:                  assert.NoError,
 			expectedCollectionPaths: []string{},
@@ -175,8 +181,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false),
 				driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
 			},
-			scope:  anyFolder,
+			inputFolderMap: map[string]string{},
-			expect: assert.NoError,
+			scope:          anyFolder,
+			expect:         assert.NoError,
 			expectedCollectionPaths: expectedPathAsSlice(
 				suite.T(),
 				tenant,
@@ -215,8 +222,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false),
 				driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
 			},
-			scope:  (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
+			inputFolderMap: map[string]string{},
-			expect: assert.NoError,
+			scope:          (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
+			expect:         assert.NoError,
 			expectedCollectionPaths: append(
 				expectedPathAsSlice(
 					suite.T(),
@@ -257,12 +265,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false),
 				driveItem("folder", "folder", testBaseDrivePath, false, true, false),
 				driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false),
-				driveItem("folder", "folder", testBaseDrivePath+folderSub, false, true, false),
+				driveItem("folder2", "folder", testBaseDrivePath+folderSub, false, true, false),
 				driveItem("package", "package", testBaseDrivePath, false, false, true),
 				driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false),
 				driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false),
 				driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
 			},
+			inputFolderMap: map[string]string{},
 			scope: (&selectors.OneDriveBackup{}).
 				Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
 			expect: assert.NoError,
@@ -276,7 +285,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			expectedFileCount:      1,
 			expectedContainerCount: 1,
 			expectedMetadataPaths: map[string]string{
-				"folder": expectedPathAsSlice(
+				"folder2": expectedPathAsSlice(
 					suite.T(),
 					tenant,
 					user,
@@ -295,8 +304,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, true, false, false),
 				driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
 			},
-			scope:  (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
+			inputFolderMap: map[string]string{},
-			expect: assert.NoError,
+			scope:          (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
+			expect:         assert.NoError,
 			expectedCollectionPaths: expectedPathAsSlice(
 				suite.T(),
 				tenant,
@@ -309,6 +319,231 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			// No child folders for subfolder so nothing here.
 			expectedMetadataPaths: map[string]string{},
 		},
+		{
+			testCase: "not moved folder tree",
+			items: []models.DriveItemable{
+				driveItem("folder", "folder", testBaseDrivePath, false, true, false),
+			},
+			inputFolderMap: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder/subfolder",
+				)[0],
+			},
+			scope:                   anyFolder,
+			expect:                  assert.NoError,
+			expectedCollectionPaths: []string{},
+			expectedItemCount:       0,
+			expectedFileCount:       0,
+			expectedContainerCount:  0,
+			expectedMetadataPaths: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder/subfolder",
+				)[0],
+			},
+		},
+		{
+			testCase: "moved folder tree",
+			items: []models.DriveItemable{
+				driveItem("folder", "folder", testBaseDrivePath, false, true, false),
+			},
+			inputFolderMap: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/a-folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/a-folder/subfolder",
+				)[0],
+			},
+			scope:                   anyFolder,
+			expect:                  assert.NoError,
+			expectedCollectionPaths: []string{},
+			expectedItemCount:       0,
+			expectedFileCount:       0,
+			expectedContainerCount:  0,
+			expectedMetadataPaths: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder/subfolder",
+				)[0],
+			},
+		},
+		{
+			testCase: "moved folder tree and subfolder 1",
+			items: []models.DriveItemable{
+				driveItem("folder", "folder", testBaseDrivePath, false, true, false),
+				driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false),
+			},
+			inputFolderMap: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/a-folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/a-folder/subfolder",
+				)[0],
+			},
+			scope:                   anyFolder,
+			expect:                  assert.NoError,
+			expectedCollectionPaths: []string{},
+			expectedItemCount:       0,
+			expectedFileCount:       0,
+			expectedContainerCount:  0,
+			expectedMetadataPaths: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/subfolder",
+				)[0],
+			},
+		},
+		{
+			testCase: "moved folder tree and subfolder 2",
+			items: []models.DriveItemable{
+				driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false),
+				driveItem("folder", "folder", testBaseDrivePath, false, true, false),
+			},
+			inputFolderMap: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/a-folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/a-folder/subfolder",
+				)[0],
+			},
+			scope:                   anyFolder,
+			expect:                  assert.NoError,
+			expectedCollectionPaths: []string{},
+			expectedItemCount:       0,
+			expectedFileCount:       0,
+			expectedContainerCount:  0,
+			expectedMetadataPaths: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/subfolder",
+				)[0],
+			},
+		},
+		{
+			testCase: "deleted folder and package",
+			items: []models.DriveItemable{
+				delItem("folder", testBaseDrivePath, false, true, false),
+				delItem("package", testBaseDrivePath, false, false, true),
+			},
+			inputFolderMap: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"package": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/package",
+				)[0],
+			},
+			scope:                   anyFolder,
+			expect:                  assert.NoError,
+			expectedCollectionPaths: []string{},
+			expectedItemCount:       0,
+			expectedFileCount:       0,
+			expectedContainerCount:  0,
+			expectedMetadataPaths:   map[string]string{},
+		},
+		{
+			testCase: "delete folder tree move subfolder",
+			items: []models.DriveItemable{
+				delItem("folder", testBaseDrivePath, false, true, false),
+				driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false),
+			},
+			inputFolderMap: map[string]string{
+				"folder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder",
+				)[0],
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/folder/subfolder",
+				)[0],
+			},
+			scope:                   anyFolder,
+			expect:                  assert.NoError,
+			expectedCollectionPaths: []string{},
+			expectedItemCount:       0,
+			expectedFileCount:       0,
+			expectedContainerCount:  0,
+			expectedMetadataPaths: map[string]string{
+				"subfolder": expectedPathAsSlice(
+					suite.T(),
+					tenant,
+					user,
+					testBaseDrivePath+"/subfolder",
+				)[0],
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -316,7 +551,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			ctx, flush := tester.NewContext()
 			defer flush()
 
-			paths := map[string]string{}
+			outputFolderMap := map[string]string{}
+			maps.Copy(outputFolderMap, tt.inputFolderMap)
 			c := NewCollections(
 				tenant,
 				user,
@@ -326,7 +562,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				nil,
 				control.Options{})
 
-			err := c.UpdateCollections(ctx, "driveID", "General", tt.items, paths)
+			err := c.UpdateCollections(ctx, "driveID", "General", tt.items, tt.inputFolderMap, outputFolderMap)
 			tt.expect(t, err)
 			assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
 			assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")
@@ -336,7 +572,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				assert.Contains(t, c.CollectionMap, collPath)
 			}
 
-			assert.Equal(t, tt.expectedMetadataPaths, paths)
+			assert.Equal(t, tt.expectedMetadataPaths, outputFolderMap)
 		})
 	}
 }
@@ -361,3 +597,26 @@ func driveItem(id string, name string, path string, isFile, isFolder, isPackage
 
 	return item
 }
+
+// delItem creates a DriveItemable that is marked as deleted. path must be set
+// to the base drive path.
+func delItem(id string, path string, isFile, isFolder, isPackage bool) models.DriveItemable {
+	item := models.NewDriveItem()
+	item.SetId(&id)
+	item.SetDeleted(models.NewDeleted())
+
+	parentReference := models.NewItemReference()
+	parentReference.SetPath(&path)
+	item.SetParentReference(parentReference)
+
+	switch {
+	case isFile:
+		item.SetFile(models.NewFile())
+	case isFolder:
+		item.SetFolder(models.NewFolder())
+	case isPackage:
+		item.SetPackage(models.NewPackage_escaped())
+	}
+
+	return item
+}

src/internal/connector/onedrive/drive.go

@@ -13,6 +13,7 @@ import (
 	"github.com/microsoftgraph/msgraph-beta-sdk-go/sites"
 	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
 	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
 
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/support"
@@ -167,7 +168,8 @@ type itemCollector func(
 	ctx context.Context,
 	driveID, driveName string,
 	driveItems []models.DriveItemable,
-	paths map[string]string,
+	oldPaths map[string]string,
+	newPaths map[string]string,
 ) error
 
 // collectItems will enumerate all items in the specified drive and hand them to the
@@ -182,9 +184,12 @@ func collectItems(
 		newDeltaURL = ""
 		// TODO(ashmrtn): Eventually this should probably be a parameter so we can
 		// take in previous paths.
-		paths = map[string]string{}
+		oldPaths = map[string]string{}
+		newPaths = map[string]string{}
 	)
 
+	maps.Copy(newPaths, oldPaths)
+
 	// TODO: Specify a timestamp in the delta query
 	// https://docs.microsoft.com/en-us/graph/api/driveitem-delta?
 	// view=graph-rest-1.0&tabs=http#example-4-retrieving-delta-results-using-a-timestamp
@@ -221,7 +226,7 @@ func collectItems(
 			)
 		}
 
-		err = collector(ctx, driveID, driveName, r.GetValue(), paths)
+		err = collector(ctx, driveID, driveName, r.GetValue(), oldPaths, newPaths)
 		if err != nil {
 			return "", nil, err
 		}
@@ -240,7 +245,7 @@ func collectItems(
 		builder = msdrives.NewItemRootDeltaRequestBuilder(*nextLink, service.Adapter())
 	}
 
-	return newDeltaURL, paths, nil
+	return newDeltaURL, newPaths, nil
 }
 
 // getFolder will lookup the specified folder name under `parentFolderID`
@@ -356,7 +361,8 @@ func GetAllFolders(
 				innerCtx context.Context,
 				driveID, driveName string,
 				items []models.DriveItemable,
-				paths map[string]string,
+				oldPaths map[string]string,
+				newPaths map[string]string,
 			) error {
 				for _, item := range items {
 					// Skip the root item.

src/internal/connector/onedrive/item_test.go

@@ -99,7 +99,8 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
 		ctx context.Context,
 		driveID, driveName string,
 		items []models.DriveItemable,
-		paths map[string]string,
+		oldPaths map[string]string,
+		newPaths map[string]string,
 	) error {
 		for _, item := range items {
 			if item.GetFile() != nil {

src/internal/connector/sharepoint/data_collections_test.go

@@ -88,6 +88,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
 			defer flush()
 
 			paths := map[string]string{}
+			newPaths := map[string]string{}
 			c := onedrive.NewCollections(
 				tenant,
 				site,
@@ -96,7 +97,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
 				&MockGraphService{},
 				nil,
 				control.Options{})
-			err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths)
+			err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths, newPaths)
 			test.expect(t, err)
 			assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
 			assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")

src/internal/kopia/upload.go

@@ -134,6 +134,7 @@ type corsoProgress struct {
 	toMerge    map[string]path.Path
 	mu         sync.RWMutex
 	totalBytes int64
+	errs       *multierror.Error
 }
 
 // Kopia interface function used as a callback when kopia finishes processing a
@@ -162,8 +163,13 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
 	// These items were sourced from a base snapshot or were cached in kopia so we
 	// never had to materialize their details in-memory.
 	if d.info == nil {
-		// TODO(ashmrtn): We should probably be returning an error here?
 		if d.prevPath == nil {
+			cp.errs = multierror.Append(cp.errs, errors.Errorf(
+				"item sourced from previous backup with no previous path. Service: %s, Category: %s",
+				d.repoPath.Service().String(),
+				d.repoPath.Category().String(),
+			))
+
 			return
 		}
 

src/internal/kopia/upload_test.go

@@ -468,7 +468,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
 
 					for k, v := range ci {
 						if cachedTest.cached {
-							cp.CachedFile(k, 42)
+							cp.CachedFile(k, v.totalBytes)
 						}
 
 						cp.FinishedFile(k, v.err)
@@ -489,6 +489,38 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
 	}
 }
 
+func (suite *CorsoProgressUnitSuite) TestFinishedFileCachedNoPrevPathErrors() {
+	t := suite.T()
+	bd := &details.Builder{}
+	cachedItems := map[string]testInfo{
+		suite.targetFileName: {
+			info:       &itemDetails{info: nil, repoPath: suite.targetFilePath},
+			err:        nil,
+			totalBytes: 100,
+		},
+	}
+	cp := corsoProgress{
+		UploadProgress: &snapshotfs.NullUploadProgress{},
+		deets:          bd,
+		pending:        map[string]*itemDetails{},
+	}
+
+	for k, v := range cachedItems {
+		cp.put(k, v.info)
+	}
+
+	require.Len(t, cp.pending, len(cachedItems))
+
+	for k, v := range cachedItems {
+		cp.CachedFile(k, v.totalBytes)
+		cp.FinishedFile(k, v.err)
+	}
+
+	assert.Empty(t, cp.pending)
+	assert.Empty(t, bd.Details().Entries)
+	assert.Error(t, cp.errs.ErrorOrNil())
+}
+
 func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchyNewItem() {
 	t := suite.T()
 	// Order of folders in hierarchy from root to leaf (excluding the item).
@@ -995,7 +1027,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() {
 						virtualfs.StreamingFileWithModTimeFromReader(
 							encodeElements(testFileName)[0],
 							time.Time{},
-							bytes.NewReader(testFileData),
+							io.NopCloser(bytes.NewReader(testFileData)),
 						),
 					},
 				),
@@ -1301,7 +1333,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
 						virtualfs.StreamingFileWithModTimeFromReader(
 							encodeElements(inboxFileName1)[0],
 							time.Time{},
-							bytes.NewReader(inboxFileData1),
+							io.NopCloser(bytes.NewReader(inboxFileData1)),
 						),
 						virtualfs.NewStaticDirectory(
 							encodeElements(personalDir)[0],
@@ -1309,12 +1341,12 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(personalFileName1)[0],
 									time.Time{},
-									bytes.NewReader(testFileData),
+									io.NopCloser(bytes.NewReader(testFileData)),
 								),
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(personalFileName2)[0],
 									time.Time{},
-									bytes.NewReader(testFileData2),
+									io.NopCloser(bytes.NewReader(testFileData2)),
 								),
 							},
 						),
@@ -1324,7 +1356,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(workFileName1)[0],
 									time.Time{},
-									bytes.NewReader(testFileData3),
+									io.NopCloser(bytes.NewReader(testFileData3)),
 								),
 							},
 						),
@@ -1941,7 +1973,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(testFileName)[0],
 									time.Time{},
-									bytes.NewReader(testFileData),
+									io.NopCloser(bytes.NewReader(testFileData)),
 								),
 							},
 						),
@@ -1951,7 +1983,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(testFileName2)[0],
 									time.Time{},
-									bytes.NewReader(testFileData2),
+									io.NopCloser(bytes.NewReader(testFileData2)),
 								),
 							},
 						),
@@ -1966,7 +1998,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(testFileName3)[0],
 									time.Time{},
-									bytes.NewReader(testFileData3),
+									io.NopCloser(bytes.NewReader(testFileData3)),
 								),
 							},
 						),
@@ -1976,7 +2008,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(testFileName4)[0],
 									time.Time{},
-									bytes.NewReader(testFileData4),
+									io.NopCloser(bytes.NewReader(testFileData4)),
 								),
 							},
 						),
@@ -2123,7 +2155,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(inboxFileName1)[0],
 									time.Time{},
-									bytes.NewReader(inboxFileData1),
+									io.NopCloser(bytes.NewReader(inboxFileData1)),
 								),
 							},
 						),
@@ -2138,7 +2170,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(contactsFileName1)[0],
 									time.Time{},
-									bytes.NewReader(contactsFileData1),
+									io.NopCloser(bytes.NewReader(contactsFileData1)),
 								),
 							},
 						),
@@ -2196,7 +2228,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
 								virtualfs.StreamingFileWithModTimeFromReader(
 									encodeElements(eventsFileName1)[0],
 									time.Time{},
-									bytes.NewReader(eventsFileData1),
+									io.NopCloser(bytes.NewReader(eventsFileData1)),
 								),
 							},
 						),

src/internal/kopia/wrapper.go

@@ -160,10 +160,11 @@ func (w Wrapper) BackupCollections(
 		progress,
 	)
 	if err != nil {
-		return nil, nil, nil, err
+		combinedErrs := multierror.Append(nil, err, progress.errs)
+		return nil, nil, nil, combinedErrs.ErrorOrNil()
 	}
 
-	return s, progress.deets, progress.toMerge, nil
+	return s, progress.deets, progress.toMerge, progress.errs.ErrorOrNil()
 }
 
 func (w Wrapper) makeSnapshotWithRoot(

src/internal/operations/backup.go

@@ -205,10 +205,21 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		return opStats.writeErr
 	}
 
+	opStats.gc = gc.AwaitStatus()
+
+	if opStats.gc.ErrorCount > 0 {
+		opStats.writeErr = multierror.Append(nil, opStats.writeErr, errors.Errorf(
+			"%v errors reported while fetching item data",
+			opStats.gc.ErrorCount,
+		)).ErrorOrNil()
+
+		// Need to exit before we set started to true else we'll report no errors.
+		return opStats.writeErr
+	}
+
 	// should always be 1, since backups are 1:1 with resourceOwners.
 	opStats.resourceCount = 1
 	opStats.started = true
-	opStats.gc = gc.AwaitStatus()
 
 	return err
 }

src/pkg/logger/logger.go

@@ -106,6 +106,11 @@ func PreloadLoggingFlags() (string, string) {
 		return "info", dlf
 	}
 
+	// if not specified, attempt to fall back to env declaration.
+	if len(logfile) == 0 {
+		logfile = os.Getenv("CORSO_LOG_FILE")
+	}
+
 	if logfile == "-" {
 		logfile = "stdout"
 	}

src/pkg/services/m365/m365.go

@@ -25,7 +25,7 @@ func Users(ctx context.Context, m365Account account.Account) ([]*User, error) {
 		return nil, errors.Wrap(err, "could not initialize M365 graph connection")
 	}
 
-	users, err := discovery.Users(ctx, gc.Service, m365Account.ID())
+	users, err := discovery.Users(ctx, gc.Owners.Users())
 	if err != nil {
 		return nil, err
 	}

website/package-lock.json

@@ -26,7 +26,7 @@
         "react-dom": "^17.0.2",
         "sass": "^1.57.1",
         "tw-elements": "^1.0.0-alpha13",
-        "wowjs": "^1.1.3"
+        "wow.js": "^1.2.2"
       },
       "devDependencies": {
         "@docusaurus/module-type-aliases": "2.2.0",
@@ -14543,13 +14543,11 @@
       "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==",
       "license": "MIT"
     },
-    "node_modules/wowjs": {
+    "node_modules/wow.js": {
-      "version": "1.1.3",
+      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz",
-      "integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==",
+      "integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A==",
-      "dependencies": {
+      "deprecated": "deprecated in favour of aos (Animate On Scroll)"
-        "animate.css": "latest"
-      }
     },
     "node_modules/wrap-ansi": {
       "version": "8.0.1",
@@ -24231,13 +24229,10 @@
       "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz",
       "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw=="
     },
-    "wowjs": {
+    "wow.js": {
-      "version": "1.1.3",
+      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz",
-      "integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==",
+      "integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A=="
-      "requires": {
-        "animate.css": "latest"
-      }
     },
     "wrap-ansi": {
       "version": "8.0.1",

website/package.json

@@ -32,7 +32,7 @@
     "react-dom": "^17.0.2",
     "sass": "^1.57.1",
     "tw-elements": "^1.0.0-alpha13",
-    "wowjs": "^1.1.3"
+    "wow.js": "^1.2.2"
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "2.2.0",

website/src/components/parts/KeyLoveFAQ.js

@@ -5,12 +5,12 @@ export default function KeyLoveFAQ() {
   const jarallaxRef = useRef(null);
   useEffect(() => {
     if (typeof window !== "undefined") {
-      const WOW = require("wowjs");
+      const WOW = require("wow.js");
       const father = require("feather-icons");
       const jarallax = require("jarallax");
       require("tw-elements");
 
-      new WOW.WOW({
+      new WOW({
         live: false,
       }).init();
       father.replace();