larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into beta-library-switch

Browse Source
This commit is contained in:
Danny Adams 2023-01-25 06:22:01 -05:00
commit ec87d58078
34 changed files with 838 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/ci.yml vendored
View File

@ -214,6 +214,7 @@ jobs:
CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
CORSO_LOG_FILE: stderr
LOG_GRAPH_REQUESTS: true
run: |
set -euo pipefail
@ -225,7 +226,7 @@ jobs:
-p 1 \
./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests
# Upload the original go test log as an artifact for later review.
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3

1
.github/workflows/load_test.yml vendored
View File

@ -53,6 +53,7 @@ jobs:
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
CORSO_LOAD_TESTS: true
CORSO_LOG_FILE: stderr
CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }}

2
CHANGELOG.md
View File

@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- msgraph-beta-sdk-go replaces msgraph-sdk-go for new features. This can lead to long build times.
- Handle case where user's drive has not been initialized
- Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163))
- Guest and External users (for cloud accounts) and non-on-premise users (for systems that use on-prem AD syncs) are now excluded from backup and restore operations.
## [v0.1.0] (alpha) - 2023-01-13

2
README.md
View File

@ -20,7 +20,7 @@ services, possibly beyond M365, will expand based on the interest and needs of t
# Getting Started
See the [Corso Documentation](https://corsobackup.io/docs/intro) for more information.
See the [Corso Quickstart](https://corsobackup.io/docs/quickstart/) on our docs page.
# Building Corso

8
src/cli/backup/exchange.go
View File

@ -61,7 +61,7 @@ const (
const (
exchangeServiceCommand = "exchange"
exchangeServiceCommandCreateUseSuffix = "--user <userId or email> | '" + utils.Wildcard + "'"
exchangeServiceCommandCreateUseSuffix = "--user <email> | '" + utils.Wildcard + "'"
exchangeServiceCommandDeleteUseSuffix = "--backup <backupId>"
exchangeServiceCommandDetailsUseSuffix = "--backup <backupId>"
)
@ -115,7 +115,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
fs.StringSliceVar(
&user,
utils.UserFN, nil,
"Backup Exchange data by user ID; accepts '"+utils.Wildcard+"' to select all users")
"Backup Exchange data by a user's email; accepts '"+utils.Wildcard+"' to select all users")
fs.StringSliceVar(
&exchangeData,
utils.DataFN, nil,
@ -274,7 +274,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
users, err := m365.UserPNs(ctx, acct)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users"))
return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 user(s)"))
}
var (
@ -347,7 +347,7 @@ func exchangeBackupCreateSelectors(userIDs, data []string) *selectors.ExchangeBa
func validateExchangeBackupCreateFlags(userIDs, data []string) error {
if len(userIDs) == 0 {
return errors.New("--user requires one or more ids or the wildcard *")
return errors.New("--user requires one or more email addresses or the wildcard '*'")
}
for _, d := range data {

4
src/cli/backup/onedrive.go
View File

@ -29,7 +29,7 @@ import (
const (
oneDriveServiceCommand = "onedrive"
oneDriveServiceCommandCreateUseSuffix = "--user <userId or email> | '" + utils.Wildcard + "'"
oneDriveServiceCommandCreateUseSuffix = "--user <email> | '" + utils.Wildcard + "'"
oneDriveServiceCommandDeleteUseSuffix = "--backup <backupId>"
oneDriveServiceCommandDetailsUseSuffix = "--backup <backupId>"
)
@ -85,7 +85,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
fs.StringSliceVar(&user,
utils.UserFN, nil,
"Backup OneDrive data by user ID; accepts '"+utils.Wildcard+"' to select all users. (required)")
"Backup OneDrive data by user's email address; accepts '"+utils.Wildcard+"' to select all users. (required)")
options.AddOperationFlags(c)
case listCommand:

53
src/cli/backup/sharepoint.go
View File

@ -27,9 +27,12 @@ import (
// setup and globals
// ------------------------------------------------------------------------------------------------
// sharePoint bucket info from flags
var (
libraryItems []string
libraryPaths []string
pageFolders []string
page []string
site []string
weburl []string
@ -38,6 +41,7 @@ var (
const (
dataLibraries = "libraries"
dataPages = "pages"
)
const (
@ -89,11 +93,10 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
utils.WebURLFN, nil,
"Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.")
// TODO: implement
fs.StringSliceVar(
&sharepointData,
utils.DataFN, nil,
"Select one or more types of data to backup: "+dataLibraries+".")
"Select one or more types of data to backup: "+dataLibraries+" or "+dataPages+".")
options.AddOperationFlags(c)
case listCommand:
@ -128,11 +131,22 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
fs.StringArrayVar(&site,
utils.SiteFN, nil,
"Backup SharePoint data by site ID; accepts '"+utils.Wildcard+"' to select all sites.")
"Select backup details by site ID; accepts '"+utils.Wildcard+"' to select all sites.")
fs.StringSliceVar(&weburl,
utils.WebURLFN, nil,
"Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.")
"Select backup data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.")
fs.StringSliceVar(
&pageFolders,
utils.PageFN, nil,
"Select backup data by site ID; accepts '"+utils.Wildcard+"' to select all sites.")
fs.StringSliceVar(
&page,
utils.PageItemFN, nil,
"Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.",
)
// info flags
@ -179,7 +193,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
return nil
}
if err := validateSharePointBackupCreateFlags(site, weburl); err != nil {
if err := validateSharePointBackupCreateFlags(site, weburl, sharepointData); err != nil {
return err
}
@ -200,7 +214,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs"))
}
sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, gc)
sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, sharepointData, gc)
if err != nil {
return Only(ctx, errors.Wrap(err, "Retrieving up sharepoint sites by ID and WebURL"))
}
@ -250,7 +264,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
return nil
}
func validateSharePointBackupCreateFlags(sites, weburls []string) error {
func validateSharePointBackupCreateFlags(sites, weburls, data []string) error {
if len(sites) == 0 && len(weburls) == 0 {
return errors.New(
"requires one or more --" +
@ -260,13 +274,21 @@ func validateSharePointBackupCreateFlags(sites, weburls []string) error {
)
}
for _, d := range data {
if d != dataLibraries && d != dataPages {
return errors.New(
d + " is an unrecognized data type; either " + dataLibraries + "or " + dataPages,
)
}
}
return nil
}
// TODO: users might specify a data type, this only supports AllData().
func sharePointBackupCreateSelectors(
ctx context.Context,
sites, weburls []string,
sites, weburls, data []string,
gc *connector.GraphConnector,
) (*selectors.SharePointBackup, error) {
if len(sites) == 0 && len(weburls) == 0 {
@ -297,7 +319,20 @@ func sharePointBackupCreateSelectors(
}
sel := selectors.NewSharePointBackup(union)
sel.Include(sel.AllData())
if len(data) == 0 {
sel.Include(sel.AllData())
return sel, nil
}
for _, d := range data {
switch d {
case dataLibraries:
sel.Include(sel.Libraries(selectors.Any()))
case dataPages:
sel.Include(sel.Pages(selectors.Any()))
}
}
return sel, nil
}

17
src/cli/backup/sharepoint_test.go
View File

@ -98,12 +98,13 @@ func (suite *SharePointSuite) TestValidateSharePointBackupCreateFlags() {
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl))
test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl, nil))
})
}
}
func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
comboString := []string{"id_1", "id_2"}
gc := &connector.GraphConnector{
Sites: map[string]string{
"url_1": "id_1",
@ -115,6 +116,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
name string
site []string
weburl []string
data []string
expect []string
expectScopesLen int
}{
@ -163,7 +165,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
name: "duplicate sites and urls",
site: []string{"id_1", "id_2"},
weburl: []string{"url_1", "url_2"},
expect: []string{"id_1", "id_2"},
expect: comboString,
expectScopesLen: 2,
},
{
@ -175,18 +177,25 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
},
{
name: "unnecessary url wildcard",
site: []string{"id_1", "id_2"},
site: comboString,
weburl: []string{"url_1", utils.Wildcard},
expect: selectors.Any(),
expectScopesLen: 2,
},
{
name: "Pages",
site: comboString,
data: []string{dataPages},
expect: comboString,
expectScopesLen: 1,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, gc)
sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, test.data, gc)
require.NoError(t, err)
assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners())

2
src/cli/restore/exchange.go
View File

@ -63,7 +63,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
fs.StringSliceVar(&user,
utils.UserFN, nil,
"Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.")
"Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.")
// email flags
fs.StringSliceVar(&email,

2
src/cli/restore/onedrive.go
View File

@ -49,7 +49,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
fs.StringSliceVar(&user,
utils.UserFN, nil,
"Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.")
"Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.")
// onedrive hierarchy (path/name) flags

30
src/go.mod
View File

@ -2,18 +2,16 @@ module github.com/alcionai/corso/src
go 1.19
replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c
require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
github.com/aws/aws-sdk-go v1.44.183
github.com/aws/aws-sdk-go v1.44.184
github.com/aws/aws-xray-sdk-go v1.8.0
github.com/google/uuid v1.3.0
github.com/hashicorp/go-multierror v1.1.1
github.com/kopia/kopia v0.12.2-0.20221229232524-ba938cf58cc8
github.com/microsoft/kiota-abstractions-go v0.16.0
github.com/microsoft/kiota-authentication-azure-go v0.6.0
github.com/microsoft/kiota-http-go v0.13.0
github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb
github.com/microsoft/kiota-abstractions-go v0.15.2
github.com/microsoft/kiota-authentication-azure-go v0.5.0
github.com/microsoft/kiota-http-go v0.11.0
github.com/microsoft/kiota-serialization-json-go v0.7.2
github.com/microsoftgraph/msgraph-beta-sdk-go v0.53.0
github.com/microsoftgraph/msgraph-sdk-go-core v0.33.0
@ -22,7 +20,7 @@ require (
github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.14.0
github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.1
github.com/tidwall/pretty v1.2.1
github.com/tomlazar/table v0.1.2
@ -40,19 +38,17 @@ require (
github.com/dnaeon/go-vcr v1.2.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/microsoft/kiota-serialization-form-go v0.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/spf13/afero v1.9.3 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.34.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
require (
@ -114,14 +110,14 @@ require (
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect
google.golang.org/grpc v1.51.0 // indirect
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
google.golang.org/grpc v1.52.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect

42
src/go.sum
View File

@ -52,8 +52,6 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c h1:uUcdEZ4sz7kRYVWB3K49MBHdICRyXCVAzd4ZiY3lvo0=
github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c/go.mod h1:yzJV11S6N6XMboXt7oCO6Jy2jJHPeSMtA+KOJ9Y1548=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@ -62,8 +60,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk=
github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo=
github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY=
github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@ -241,6 +239,8 @@ github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747h
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kopia/htmluibuild v0.0.0-20220928042710-9fdd02afb1e7 h1:WP5VfIQL7AaYkO4zTNSCsVOawTzudbc4tvLojvg0RKc=
github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb h1:0jLaKLiloYvRNbuHHpnQkJ7STAgzQ4z6n+KPa6Kyg7I=
github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb/go.mod h1:dtCyMCsWulG82o9bDopvnny9DpOQe0PnSDczJLuhnWA=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -251,8 +251,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
@ -303,10 +303,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg=
github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas=
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
@ -362,8 +360,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 h1:lQ3JvmcVO1/AMFbabvUSJ4YtJRpEAX9Qza73p5j03sw=
github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1/go.mod h1:4aKqcbhASNqjbrG0h9BmkzcWvPJGxbef4B+j0XfFrZo=
github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw=
github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
@ -372,8 +370,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU=
github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As=
github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@ -387,8 +385,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
github.com/tg123/go-htpasswd v1.2.0 h1:UKp34m9H467/xklxUxU15wKRru7fwXoTojtxg25ITF0=
github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@ -450,8 +448,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -747,8 +745,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc=
google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0=
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY=
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@ -765,8 +763,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk=
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

8
src/internal/connector/data_collections.go
View File

@ -8,8 +8,8 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/discovery"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/connector/support"
@ -44,7 +44,7 @@ func (gc *GraphConnector) DataCollections(
return nil, err
}
serviceEnabled, err := checkServiceEnabled(ctx, gc.Service, path.ServiceType(sels.Service), sels.DiscreteOwner)
serviceEnabled, err := checkServiceEnabled(ctx, gc.Owners.Users(), path.ServiceType(sels.Service), sels.DiscreteOwner)
if err != nil {
return nil, err
}
@ -138,7 +138,7 @@ func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) erro
func checkServiceEnabled(
ctx context.Context,
gs graph.Servicer,
au api.Users,
service path.ServiceType,
resource string,
) (bool, error) {
@ -147,7 +147,7 @@ func checkServiceEnabled(
return true, nil
}
_, info, err := discovery.User(ctx, gs, resource)
_, info, err := discovery.User(ctx, au, resource)
if err != nil {
return false, err
}

57
src/internal/connector/discovery/api/api.go Normal file
View File

@ -0,0 +1,57 @@
package api
import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
)
// ---------------------------------------------------------------------------
// interfaces
// ---------------------------------------------------------------------------
// Client is used to fulfill the interface for discovery
// queries that are traditionally backed by GraphAPI. A
// struct is used in this case, instead of deferring to
// pure function wrappers, so that the boundary separates the
// granular implementation of the graphAPI and kiota away
// from the exchange package's broader intents.
type Client struct {
Credentials account.M365Config
// The stable service is re-usable for any non-paged request.
// This allows us to maintain performance across async requests.
stable graph.Servicer
}
// NewClient produces a new exchange api client. Must be used in
// place of creating an ad-hoc client struct.
func NewClient(creds account.M365Config) (Client, error) {
s, err := newService(creds)
if err != nil {
return Client{}, err
}
return Client{creds, s}, nil
}
// service generates a new service. Used for paged and other long-running
// requests instead of the client's stable service, so that in-flight state
// within the adapter doesn't get clobbered
func (c Client) service() (*graph.Service, error) {
return newService(c.Credentials)
}
func newService(creds account.M365Config) (*graph.Service, error) {
adapter, err := graph.CreateAdapter(
creds.AzureTenantID,
creds.AzureClientID,
creds.AzureClientSecret,
)
if err != nil {
return nil, errors.Wrap(err, "generating graph api service client")
}
return graph.NewService(adapter), nil
}

166
src/internal/connector/discovery/api/users.go Normal file
View File

@ -0,0 +1,166 @@
package api
import (
"context"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
)
// ---------------------------------------------------------------------------
// controller
// ---------------------------------------------------------------------------
func (c Client) Users() Users {
return Users{c}
}
// Users is an interface-compliant provider of the client.
type Users struct {
Client
}
// ---------------------------------------------------------------------------
// structs
// ---------------------------------------------------------------------------
type UserInfo struct {
DiscoveredServices map[path.ServiceType]struct{}
}
func newUserInfo() *UserInfo {
return &UserInfo{
DiscoveredServices: map[path.ServiceType]struct{}{
path.ExchangeService: {},
path.OneDriveService: {},
},
}
}
// ---------------------------------------------------------------------------
// methods
// ---------------------------------------------------------------------------
const (
userSelectID = "id"
userSelectPrincipalName = "userPrincipalName"
userSelectDisplayName = "displayName"
)
// Filter out both guest users, and (for on-prem installations) non-synced users.
// The latter filter makes an assumption that no on-prem users are guests; this might
// require more fine-tuned controls in the future.
// https://stackoverflow.com/questions/64044266/error-message-unsupported-or-invalid-query-filter-clause-specified-for-property
//
//nolint:lll
var userFilterNoGuests = "onPremisesSyncEnabled eq true OR userType eq 'Member'"
func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration {
return &users.UsersRequestBuilderGetRequestConfiguration{
QueryParameters: &users.UsersRequestBuilderGetQueryParameters{
Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName},
Filter: fs,
},
}
}
// GetAll retrieves all users.
func (c Users) GetAll(ctx context.Context) ([]models.Userable, error) {
service, err := c.service()
if err != nil {
return nil, err
}
resp, err := service.Client().Users().Get(ctx, userOptions(&userFilterNoGuests))
if err != nil {
return nil, support.ConnectorStackErrorTraceWrap(err, "getting all users")
}
iter, err := msgraphgocore.NewPageIterator(
resp,
service.Adapter(),
models.CreateUserCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, support.ConnectorStackErrorTraceWrap(err, "constructing user iterator")
}
var (
iterErrs error
us = make([]models.Userable, 0)
)
iterator := func(item any) bool {
u, err := validateUser(item)
if err != nil {
iterErrs = support.WrapAndAppend("validating user", err, iterErrs)
} else {
us = append(us, u)
}
return true
}
if err := iter.Iterate(ctx, iterator); err != nil {
return nil, support.ConnectorStackErrorTraceWrap(err, "iterating all users")
}
return us, iterErrs
}
func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, error) {
user, err := c.stable.Client().UsersById(userID).Get(ctx, nil)
if err != nil {
return nil, support.ConnectorStackErrorTraceWrap(err, "getting user by id")
}
return user, nil
}
func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) {
// Assume all services are enabled
// then filter down to only services the user has enabled
userInfo := newUserInfo()
// TODO: OneDrive
_, err := c.stable.Client().UsersById(userID).MailFolders().Get(ctx, nil)
if err != nil {
if !graph.IsErrExchangeMailFolderNotFound(err) {
return nil, support.ConnectorStackErrorTraceWrap(err, "getting user's exchange mailfolders")
}
delete(userInfo.DiscoveredServices, path.ExchangeService)
}
return userInfo, nil
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
// validateUser ensures the item is a Userable, and contains the necessary
// identifiers that we handle with all users.
// returns the item as a Userable model.
func validateUser(item any) (models.Userable, error) {
m, ok := item.(models.Userable)
if !ok {
return nil, errors.Errorf("expected Userable, got %T", item)
}
if m.GetId() == nil {
return nil, errors.Errorf("missing ID")
}
if m.GetUserPrincipalName() == nil {
return nil, errors.New("missing principalName")
}
return m, nil
}

12
src/internal/connector/discovery/discovery_test.go → src/internal/connector/discovery/api/users_test.go
View File

@ -1,4 +1,4 @@
package discovery
package api
import (
"reflect"
@ -8,15 +8,15 @@ import (
"github.com/stretchr/testify/suite"
)
type DiscoverySuite struct {
type UsersUnitSuite struct {
suite.Suite
}
func TestDiscoverySuite(t *testing.T) {
suite.Run(t, new(DiscoverySuite))
func TestUsersUnitSuite(t *testing.T) {
suite.Run(t, new(UsersUnitSuite))
}
func (suite *DiscoverySuite) TestParseUser() {
func (suite *UsersUnitSuite) TestValidateUser() {
t := suite.T()
name := "testuser"
@ -60,7 +60,7 @@ func (suite *DiscoverySuite) TestParseUser() {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseUser(tt.args)
got, err := validateUser(tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("parseUser() error = %v, wantErr %v", err, tt.wantErr)
return

139
src/internal/connector/discovery/discovery.go
View File

@ -3,125 +3,52 @@ package discovery
import (
"context"
"github.com/microsoftgraph/msgraph-beta-sdk-go/models"
msuser "github.com/microsoftgraph/msgraph-beta-sdk-go/users"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
)
const (
userSelectID = "id"
userSelectPrincipalName = "userPrincipalName"
userSelectDisplayName = "displayName"
)
// ---------------------------------------------------------------------------
// interfaces
// ---------------------------------------------------------------------------
func Users(ctx context.Context, gs graph.Servicer, tenantID string) ([]models.Userable, error) {
users := make([]models.Userable, 0)
options := &msuser.UsersRequestBuilderGetRequestConfiguration{
QueryParameters: &msuser.UsersRequestBuilderGetQueryParameters{
Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName},
},
}
response, err := gs.Client().Users().Get(ctx, options)
if err != nil {
return nil, errors.Wrapf(
err,
"retrieving resources for tenant %s: %s",
tenantID,
support.ConnectorStackErrorTrace(err),
)
}
iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(),
models.CreateUserCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
var iterErrs error
callbackFunc := func(item interface{}) bool {
u, err := parseUser(item)
if err != nil {
iterErrs = support.WrapAndAppend("discovering users: ", err, iterErrs)
return true
}
users = append(users, u)
return true
}
if err := iter.Iterate(ctx, callbackFunc); err != nil {
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
return users, iterErrs
type getAller interface {
GetAll(context.Context) ([]models.Userable, error)
}
type UserInfo struct {
DiscoveredServices map[path.ServiceType]struct{}
type getter interface {
GetByID(context.Context, string) (models.Userable, error)
}
func User(ctx context.Context, gs graph.Servicer, userID string) (models.Userable, *UserInfo, error) {
user, err := gs.Client().UsersById(userID).Get(ctx, nil)
type getInfoer interface {
GetInfo(context.Context, string) (*api.UserInfo, error)
}
type getWithInfoer interface {
getter
getInfoer
}
// ---------------------------------------------------------------------------
// api
// ---------------------------------------------------------------------------
// Users fetches all users in the tenant.
func Users(ctx context.Context, ga getAller) ([]models.Userable, error) {
return ga.GetAll(ctx)
}
func User(ctx context.Context, gwi getWithInfoer, userID string) (models.Userable, *api.UserInfo, error) {
u, err := gwi.GetByID(ctx, userID)
if err != nil {
return nil, nil, errors.Wrapf(
err,
"retrieving resource for tenant: %s",
support.ConnectorStackErrorTrace(err),
)
return nil, nil, errors.Wrap(err, "getting user")
}
// Assume all services are enabled
userInfo := &UserInfo{
DiscoveredServices: map[path.ServiceType]struct{}{
path.ExchangeService: {},
path.OneDriveService: {},
},
}
// Discover which services the user has enabled
// Exchange: Query `MailFolders`
_, err = gs.Client().UsersById(userID).MailFolders().Get(ctx, nil)
ui, err := gwi.GetInfo(ctx, userID)
if err != nil {
if !graph.IsErrExchangeMailFolderNotFound(err) {
return nil, nil, errors.Wrapf(
err,
"retrieving mail folders for tenant: %s",
support.ConnectorStackErrorTrace(err),
)
}
delete(userInfo.DiscoveredServices, path.ExchangeService)
return nil, nil, errors.Wrap(err, "getting user info")
}
// TODO: OneDrive
return user, userInfo, nil
}
// parseUser extracts information from `models.Userable` we care about
func parseUser(item interface{}) (models.Userable, error) {
m, ok := item.(models.Userable)
if !ok {
return nil, errors.New("iteration retrieved non-User item")
}
if m.GetId() == nil {
return nil, errors.Errorf("no ID for User")
}
if m.GetUserPrincipalName() == nil {
return nil, errors.Errorf("no principal name for User: %s", *m.GetId())
}
return m, nil
return u, ui, nil
}

28
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -257,6 +257,24 @@ func (col *Collection) streamItems(ctx context.Context) {
break
}
// If the data is no longer available just return here and chalk it up
// as a success. There's no reason to retry and no way we can backup up
// enough information to restore the item anyway.
if e := graph.IsErrDeletedInFlight(err); e != nil {
atomic.AddInt64(&success, 1)
logger.Ctx(ctx).Infow(
"Graph reported item not found",
"error",
e,
"service",
path.ExchangeService.String(),
"category",
col.category.String,
)
return
}
if i < numberOfRetries {
time.Sleep(time.Duration(3*(i+1)) * time.Second)
}
@ -270,6 +288,16 @@ func (col *Collection) streamItems(ctx context.Context) {
// attempted items.
if e := graph.IsErrDeletedInFlight(err); e != nil {
atomic.AddInt64(&success, 1)
logger.Ctx(ctx).Infow(
"Graph reported item not found",
"error",
e,
"service",
path.ExchangeService.String(),
"category",
col.category.String,
)
return
}

14
src/internal/connector/graph_connector.go
View File

@ -15,6 +15,7 @@ import (
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/connector/discovery"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
@ -37,7 +38,9 @@ import (
// GraphRequestAdapter from the msgraph-sdk-go. Additional fields are for
// bookkeeping and interfacing with other component.
type GraphConnector struct {
Service graph.Servicer
Service graph.Servicer
Owners api.Client
tenant string
Users map[string]string // key<email> value<id>
Sites map[string]string // key<???> value<???>
@ -74,12 +77,15 @@ func NewGraphConnector(ctx context.Context, acct account.Account, r resource) (*
credentials: m365,
}
gService, err := gc.createService()
gc.Service, err = gc.createService()
if err != nil {
return nil, errors.Wrap(err, "creating service connection")
}
gc.Service = gService
gc.Owners, err = api.NewClient(m365)
if err != nil {
return nil, errors.Wrap(err, "creating api client")
}
// TODO(ashmrtn): When selectors only encapsulate a single resource owner that
// is not a wildcard don't populate users or sites when making the connector.
@ -121,7 +127,7 @@ func (gc *GraphConnector) setTenantUsers(ctx context.Context) error {
ctx, end := D.Span(ctx, "gc:setTenantUsers")
defer end()
users, err := discovery.Users(ctx, gc.Service, gc.tenant)
users, err := discovery.Users(ctx, gc.Owners.Users())
if err != nil {
return err
}

5
src/internal/connector/graph_connector_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/suite"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
@ -174,10 +175,10 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
ctx, flush := tester.NewContext()
defer flush()
service, err := newConnector.createService()
owners, err := api.NewClient(suite.connector.credentials)
require.NoError(suite.T(), err)
newConnector.Service = service
newConnector.Owners = owners
suite.Empty(len(newConnector.Users))
err = newConnector.setTenantUsers(ctx)

63
src/internal/connector/onedrive/collections.go
View File

@ -161,12 +161,16 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
}
// UpdateCollections initializes and adds the provided drive items to Collections
// A new collection is created for every drive folder (or package)
// A new collection is created for every drive folder (or package).
// oldPaths is the unchanged data that was loaded from the metadata file.
// newPaths starts as a copy of oldPaths and is updated as changes are found in
// the returned results.
func (c *Collections) UpdateCollections(
ctx context.Context,
driveID, driveName string,
items []models.DriveItemable,
paths map[string]string,
oldPaths map[string]string,
newPaths map[string]string,
) error {
for _, item := range items {
if item.GetRoot() != nil {
@ -197,23 +201,40 @@ func (c *Collections) UpdateCollections(
switch {
case item.GetFolder() != nil, item.GetPackage() != nil:
// Eventually, deletions of folders will be handled here so we may as well
// start off by saving the path.Path of the item instead of just the
// OneDrive parentRef or such.
if item.GetDeleted() != nil {
// Nested folders also return deleted delta results so we don't have to
// worry about doing a prefix search in the map to remove the subtree of
// the deleted folder/package.
delete(newPaths, *item.GetId())
// TODO(ashmrtn): Create a collection with state Deleted.
break
}
// Deletions of folders are handled in this case so we may as well start
// off by saving the path.Path of the item instead of just the OneDrive
// parentRef or such.
folderPath, err := collectionPath.Append(*item.GetName(), false)
if err != nil {
logger.Ctx(ctx).Errorw("failed building collection path", "error", err)
return err
}
// TODO(ashmrtn): Handle deletions by removing this entry from the map.
// TODO(ashmrtn): Handle moves by setting the collection state if the
// collection doesn't already exist/have that state.
paths[*item.GetId()] = folderPath.String()
// Moved folders don't cause delta results for any subfolders nested in
// them. We need to go through and update paths to handle that. We only
// update newPaths so we don't accidentally clobber previous deletes.
//
// TODO(ashmrtn): Since we're also getting notifications about folder
// moves we may need to handle updates to a path of a collection we've
// already created and partially populated.
updatePath(newPaths, *item.GetId(), folderPath.String())
case item.GetFile() != nil:
col, found := c.CollectionMap[collectionPath.String()]
if !found {
// TODO(ashmrtn): Compare old and new path and set collection state
// accordingly.
col = NewCollection(
collectionPath,
driveID,
@ -286,3 +307,27 @@ func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) boo
return m.Matches(folderPathString)
}
func updatePath(paths map[string]string, id, newPath string) {
oldPath := paths[id]
if len(oldPath) == 0 {
paths[id] = newPath
return
}
if oldPath == newPath {
return
}
// We need to do a prefix search on the rest of the map to update the subtree.
// We don't need to make collections for all of these, as hierarchy merging in
// other components should take care of that. We do need to ensure that the
// resulting map contains all folders though so we know the next time around.
for folderID, p := range paths {
if !strings.HasPrefix(p, oldPath) {
continue
}
paths[folderID] = strings.Replace(p, oldPath, newPath, 1)
}
}

285
src/internal/connector/onedrive/collections_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control"
@ -96,6 +97,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
tests := []struct {
testCase string
items []models.DriveItemable
inputFolderMap map[string]string
scope selectors.OneDriveScope
expect assert.ErrorAssertionFunc
expectedCollectionPaths []string
@ -109,6 +111,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
items: []models.DriveItemable{
driveItem("item", "item", testBaseDrivePath, false, false, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.Error,
expectedMetadataPaths: map[string]string{},
@ -118,8 +121,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath, true, false, false),
},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: expectedPathAsSlice(
suite.T(),
tenant,
@ -137,6 +141,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
@ -154,6 +159,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
items: []models.DriveItemable{
driveItem("package", "package", testBaseDrivePath, false, false, true),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
@ -175,8 +181,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: expectedPathAsSlice(
suite.T(),
tenant,
@ -215,8 +222,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false),
driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
expect: assert.NoError,
expectedCollectionPaths: append(
expectedPathAsSlice(
suite.T(),
@ -257,12 +265,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false),
driveItem("folder", "folder", testBaseDrivePath, false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false),
driveItem("folder", "folder", testBaseDrivePath+folderSub, false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folderSub, false, true, false),
driveItem("package", "package", testBaseDrivePath, false, false, true),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false),
driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).
Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
expect: assert.NoError,
@ -276,7 +285,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
expectedFileCount: 1,
expectedContainerCount: 1,
expectedMetadataPaths: map[string]string{
"folder": expectedPathAsSlice(
"folder2": expectedPathAsSlice(
suite.T(),
tenant,
user,
@ -295,8 +304,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
expect: assert.NoError,
expectedCollectionPaths: expectedPathAsSlice(
suite.T(),
tenant,
@ -309,6 +319,231 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
// No child folders for subfolder so nothing here.
expectedMetadataPaths: map[string]string{},
},
{
testCase: "not moved folder tree",
items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false),
},
inputFolderMap: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder/subfolder",
)[0],
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
expectedItemCount: 0,
expectedFileCount: 0,
expectedContainerCount: 0,
expectedMetadataPaths: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder/subfolder",
)[0],
},
},
{
testCase: "moved folder tree",
items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false),
},
inputFolderMap: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/a-folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/a-folder/subfolder",
)[0],
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
expectedItemCount: 0,
expectedFileCount: 0,
expectedContainerCount: 0,
expectedMetadataPaths: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder/subfolder",
)[0],
},
},
{
testCase: "moved folder tree and subfolder 1",
items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false),
},
inputFolderMap: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/a-folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/a-folder/subfolder",
)[0],
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
expectedItemCount: 0,
expectedFileCount: 0,
expectedContainerCount: 0,
expectedMetadataPaths: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/subfolder",
)[0],
},
},
{
testCase: "moved folder tree and subfolder 2",
items: []models.DriveItemable{
driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false),
driveItem("folder", "folder", testBaseDrivePath, false, true, false),
},
inputFolderMap: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/a-folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/a-folder/subfolder",
)[0],
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
expectedItemCount: 0,
expectedFileCount: 0,
expectedContainerCount: 0,
expectedMetadataPaths: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/subfolder",
)[0],
},
},
{
testCase: "deleted folder and package",
items: []models.DriveItemable{
delItem("folder", testBaseDrivePath, false, true, false),
delItem("package", testBaseDrivePath, false, false, true),
},
inputFolderMap: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"package": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/package",
)[0],
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
expectedItemCount: 0,
expectedFileCount: 0,
expectedContainerCount: 0,
expectedMetadataPaths: map[string]string{},
},
{
testCase: "delete folder tree move subfolder",
items: []models.DriveItemable{
delItem("folder", testBaseDrivePath, false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false),
},
inputFolderMap: map[string]string{
"folder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0],
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder/subfolder",
)[0],
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: []string{},
expectedItemCount: 0,
expectedFileCount: 0,
expectedContainerCount: 0,
expectedMetadataPaths: map[string]string{
"subfolder": expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/subfolder",
)[0],
},
},
}
for _, tt := range tests {
@ -316,7 +551,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
ctx, flush := tester.NewContext()
defer flush()
paths := map[string]string{}
outputFolderMap := map[string]string{}
maps.Copy(outputFolderMap, tt.inputFolderMap)
c := NewCollections(
tenant,
user,
@ -326,7 +562,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
nil,
control.Options{})
err := c.UpdateCollections(ctx, "driveID", "General", tt.items, paths)
err := c.UpdateCollections(ctx, "driveID", "General", tt.items, tt.inputFolderMap, outputFolderMap)
tt.expect(t, err)
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")
@ -336,7 +572,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
assert.Contains(t, c.CollectionMap, collPath)
}
assert.Equal(t, tt.expectedMetadataPaths, paths)
assert.Equal(t, tt.expectedMetadataPaths, outputFolderMap)
})
}
}
@ -361,3 +597,26 @@ func driveItem(id string, name string, path string, isFile, isFolder, isPackage
return item
}
// delItem creates a DriveItemable that is marked as deleted. path must be set
// to the base drive path.
func delItem(id string, path string, isFile, isFolder, isPackage bool) models.DriveItemable {
item := models.NewDriveItem()
item.SetId(&id)
item.SetDeleted(models.NewDeleted())
parentReference := models.NewItemReference()
parentReference.SetPath(&path)
item.SetParentReference(parentReference)
switch {
case isFile:
item.SetFile(models.NewFile())
case isFolder:
item.SetFolder(models.NewFolder())
case isPackage:
item.SetPackage(models.NewPackage_escaped())
}
return item
}

16
src/internal/connector/onedrive/drive.go
View File

@ -13,6 +13,7 @@ import (
"github.com/microsoftgraph/msgraph-beta-sdk-go/sites"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
@ -167,7 +168,8 @@ type itemCollector func(
ctx context.Context,
driveID, driveName string,
driveItems []models.DriveItemable,
paths map[string]string,
oldPaths map[string]string,
newPaths map[string]string,
) error
// collectItems will enumerate all items in the specified drive and hand them to the
@ -182,9 +184,12 @@ func collectItems(
newDeltaURL = ""
// TODO(ashmrtn): Eventually this should probably be a parameter so we can
// take in previous paths.
paths = map[string]string{}
oldPaths = map[string]string{}
newPaths = map[string]string{}
)
maps.Copy(newPaths, oldPaths)
// TODO: Specify a timestamp in the delta query
// https://docs.microsoft.com/en-us/graph/api/driveitem-delta?
// view=graph-rest-1.0&tabs=http#example-4-retrieving-delta-results-using-a-timestamp
@ -221,7 +226,7 @@ func collectItems(
)
}
err = collector(ctx, driveID, driveName, r.GetValue(), paths)
err = collector(ctx, driveID, driveName, r.GetValue(), oldPaths, newPaths)
if err != nil {
return "", nil, err
}
@ -240,7 +245,7 @@ func collectItems(
builder = msdrives.NewItemRootDeltaRequestBuilder(*nextLink, service.Adapter())
}
return newDeltaURL, paths, nil
return newDeltaURL, newPaths, nil
}
// getFolder will lookup the specified folder name under `parentFolderID`
@ -356,7 +361,8 @@ func GetAllFolders(
innerCtx context.Context,
driveID, driveName string,
items []models.DriveItemable,
paths map[string]string,
oldPaths map[string]string,
newPaths map[string]string,
) error {
for _, item := range items {
// Skip the root item.

3
src/internal/connector/onedrive/item_test.go
View File

@ -99,7 +99,8 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
ctx context.Context,
driveID, driveName string,
items []models.DriveItemable,
paths map[string]string,
oldPaths map[string]string,
newPaths map[string]string,
) error {
for _, item := range items {
if item.GetFile() != nil {

3
src/internal/connector/sharepoint/data_collections_test.go
View File

@ -88,6 +88,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
defer flush()
paths := map[string]string{}
newPaths := map[string]string{}
c := onedrive.NewCollections(
tenant,
site,
@ -96,7 +97,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
&MockGraphService{},
nil,
control.Options{})
err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths)
err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths, newPaths)
test.expect(t, err)
assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")

8
src/internal/kopia/upload.go
View File

@ -134,6 +134,7 @@ type corsoProgress struct {
toMerge map[string]path.Path
mu sync.RWMutex
totalBytes int64
errs *multierror.Error
}
// Kopia interface function used as a callback when kopia finishes processing a
@ -162,8 +163,13 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
// These items were sourced from a base snapshot or were cached in kopia so we
// never had to materialize their details in-memory.
if d.info == nil {
// TODO(ashmrtn): We should probably be returning an error here?
if d.prevPath == nil {
cp.errs = multierror.Append(cp.errs, errors.Errorf(
"item sourced from previous backup with no previous path. Service: %s, Category: %s",
d.repoPath.Service().String(),
d.repoPath.Category().String(),
))
return
}

58
src/internal/kopia/upload_test.go
View File

@ -468,7 +468,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
for k, v := range ci {
if cachedTest.cached {
cp.CachedFile(k, 42)
cp.CachedFile(k, v.totalBytes)
}
cp.FinishedFile(k, v.err)
@ -489,6 +489,38 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
}
}
func (suite *CorsoProgressUnitSuite) TestFinishedFileCachedNoPrevPathErrors() {
t := suite.T()
bd := &details.Builder{}
cachedItems := map[string]testInfo{
suite.targetFileName: {
info: &itemDetails{info: nil, repoPath: suite.targetFilePath},
err: nil,
totalBytes: 100,
},
}
cp := corsoProgress{
UploadProgress: &snapshotfs.NullUploadProgress{},
deets: bd,
pending: map[string]*itemDetails{},
}
for k, v := range cachedItems {
cp.put(k, v.info)
}
require.Len(t, cp.pending, len(cachedItems))
for k, v := range cachedItems {
cp.CachedFile(k, v.totalBytes)
cp.FinishedFile(k, v.err)
}
assert.Empty(t, cp.pending)
assert.Empty(t, bd.Details().Entries)
assert.Error(t, cp.errs.ErrorOrNil())
}
func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchyNewItem() {
t := suite.T()
// Order of folders in hierarchy from root to leaf (excluding the item).
@ -995,7 +1027,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() {
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName)[0],
time.Time{},
bytes.NewReader(testFileData),
io.NopCloser(bytes.NewReader(testFileData)),
),
},
),
@ -1301,7 +1333,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(inboxFileName1)[0],
time.Time{},
bytes.NewReader(inboxFileData1),
io.NopCloser(bytes.NewReader(inboxFileData1)),
),
virtualfs.NewStaticDirectory(
encodeElements(personalDir)[0],
@ -1309,12 +1341,12 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(personalFileName1)[0],
time.Time{},
bytes.NewReader(testFileData),
io.NopCloser(bytes.NewReader(testFileData)),
),
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(personalFileName2)[0],
time.Time{},
bytes.NewReader(testFileData2),
io.NopCloser(bytes.NewReader(testFileData2)),
),
},
),
@ -1324,7 +1356,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(workFileName1)[0],
time.Time{},
bytes.NewReader(testFileData3),
io.NopCloser(bytes.NewReader(testFileData3)),
),
},
),
@ -1941,7 +1973,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName)[0],
time.Time{},
bytes.NewReader(testFileData),
io.NopCloser(bytes.NewReader(testFileData)),
),
},
),
@ -1951,7 +1983,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName2)[0],
time.Time{},
bytes.NewReader(testFileData2),
io.NopCloser(bytes.NewReader(testFileData2)),
),
},
),
@ -1966,7 +1998,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName3)[0],
time.Time{},
bytes.NewReader(testFileData3),
io.NopCloser(bytes.NewReader(testFileData3)),
),
},
),
@ -1976,7 +2008,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName4)[0],
time.Time{},
bytes.NewReader(testFileData4),
io.NopCloser(bytes.NewReader(testFileData4)),
),
},
),
@ -2123,7 +2155,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(inboxFileName1)[0],
time.Time{},
bytes.NewReader(inboxFileData1),
io.NopCloser(bytes.NewReader(inboxFileData1)),
),
},
),
@ -2138,7 +2170,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(contactsFileName1)[0],
time.Time{},
bytes.NewReader(contactsFileData1),
io.NopCloser(bytes.NewReader(contactsFileData1)),
),
},
),
@ -2196,7 +2228,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(eventsFileName1)[0],
time.Time{},
bytes.NewReader(eventsFileData1),
io.NopCloser(bytes.NewReader(eventsFileData1)),
),
},
),

5
src/internal/kopia/wrapper.go
View File

@ -160,10 +160,11 @@ func (w Wrapper) BackupCollections(
progress,
)
if err != nil {
return nil, nil, nil, err
combinedErrs := multierror.Append(nil, err, progress.errs)
return nil, nil, nil, combinedErrs.ErrorOrNil()
}
return s, progress.deets, progress.toMerge, nil
return s, progress.deets, progress.toMerge, progress.errs.ErrorOrNil()
}
func (w Wrapper) makeSnapshotWithRoot(

13
src/internal/operations/backup.go
View File

@ -205,10 +205,21 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
return opStats.writeErr
}
opStats.gc = gc.AwaitStatus()
if opStats.gc.ErrorCount > 0 {
opStats.writeErr = multierror.Append(nil, opStats.writeErr, errors.Errorf(
"%v errors reported while fetching item data",
opStats.gc.ErrorCount,
)).ErrorOrNil()
// Need to exit before we set started to true else we'll report no errors.
return opStats.writeErr
}
// should always be 1, since backups are 1:1 with resourceOwners.
opStats.resourceCount = 1
opStats.started = true
opStats.gc = gc.AwaitStatus()
return err
}

5
src/pkg/logger/logger.go
View File

@ -106,6 +106,11 @@ func PreloadLoggingFlags() (string, string) {
return "info", dlf
}
// if not specified, attempt to fall back to env declaration.
if len(logfile) == 0 {
logfile = os.Getenv("CORSO_LOG_FILE")
}
if logfile == "-" {
logfile = "stdout"
}

2
src/pkg/services/m365/m365.go
View File

@ -25,7 +25,7 @@ func Users(ctx context.Context, m365Account account.Account) ([]*User, error) {
return nil, errors.Wrap(err, "could not initialize M365 graph connection")
}
users, err := discovery.Users(ctx, gc.Service, m365Account.ID())
users, err := discovery.Users(ctx, gc.Owners.Users())
if err != nil {
return nil, err
}

25
website/package-lock.json generated
View File

@ -26,7 +26,7 @@
"react-dom": "^17.0.2",
"sass": "^1.57.1",
"tw-elements": "^1.0.0-alpha13",
"wowjs": "^1.1.3"
"wow.js": "^1.2.2"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "2.2.0",
@ -14543,13 +14543,11 @@
"integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==",
"license": "MIT"
},
"node_modules/wowjs": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz",
"integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==",
"dependencies": {
"animate.css": "latest"
}
"node_modules/wow.js": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz",
"integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A==",
"deprecated": "deprecated in favour of aos (Animate On Scroll)"
},
"node_modules/wrap-ansi": {
"version": "8.0.1",
@ -24231,13 +24229,10 @@
"resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz",
"integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw=="
},
"wowjs": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz",
"integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==",
"requires": {
"animate.css": "latest"
}
"wow.js": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz",
"integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A=="
},
"wrap-ansi": {
"version": "8.0.1",

2
website/package.json
View File

@ -32,7 +32,7 @@
"react-dom": "^17.0.2",
"sass": "^1.57.1",
"tw-elements": "^1.0.0-alpha13",
"wowjs": "^1.1.3"
"wow.js": "^1.2.2"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "2.2.0",

4
website/src/components/parts/KeyLoveFAQ.js
View File

@ -5,12 +5,12 @@ export default function KeyLoveFAQ() {
const jarallaxRef = useRef(null);
useEffect(() => {
if (typeof window !== "undefined") {
const WOW = require("wowjs");
const WOW = require("wow.js");
const father = require("feather-icons");
const jarallax = require("jarallax");
require("tw-elements");
new WOW.WOW({
new WOW({
live: false,
}).init();
father.replace();