From 6651305d4fb2f0edcca6300ceaa64fdf5f107a44 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Fri, 20 Jan 2023 09:02:23 -0800 Subject: [PATCH 01/16] Update OneDrive metadata with folder deletions (#2192) ## Description Track folder and package deletions in OneDrive metadata for incremental backups. Add test for deletions ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * #2120 ## Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/onedrive/collections.go | 18 +++- .../connector/onedrive/collections_test.go | 88 ++++++++++++++++--- 2 files changed, 89 insertions(+), 17 deletions(-) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 6a59104f1..73e1b1ba8 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -197,16 +197,26 @@ func (c *Collections) UpdateCollections( switch { case item.GetFolder() != nil, item.GetPackage() != nil: - // Eventually, deletions of folders will be handled here so we may as well - // start off by saving the path.Path of the item instead of just the - // OneDrive parentRef or such. + if item.GetDeleted() != nil { + // Nested folders also return deleted delta results so we don't have to + // worry about doing a prefix search in the map to remove the subtree of + // the deleted folder/package. + delete(paths, *item.GetId()) + + // TODO(ashmrtn): Create a collection with state Deleted. + + break + } + + // Deletions of folders are handled in this case so we may as well start + // off by saving the path.Path of the item instead of just the OneDrive + // parentRef or such. folderPath, err := collectionPath.Append(*item.GetName(), false) if err != nil { logger.Ctx(ctx).Errorw("failed building collection path", "error", err) return err } - // TODO(ashmrtn): Handle deletions by removing this entry from the map. // TODO(ashmrtn): Handle moves by setting the collection state if the // collection doesn't already exist/have that state. paths[*item.GetId()] = folderPath.String() diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 5a0775edc..a8520d52d 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" @@ -96,6 +97,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { tests := []struct { testCase string items []models.DriveItemable + inputFolderMap map[string]string scope selectors.OneDriveScope expect assert.ErrorAssertionFunc expectedCollectionPaths []string @@ -109,6 +111,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("item", "item", testBaseDrivePath, false, false, false), }, + inputFolderMap: map[string]string{}, scope: anyFolder, expect: assert.Error, expectedMetadataPaths: map[string]string{}, @@ -118,8 +121,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("file", "file", testBaseDrivePath, true, false, false), }, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, @@ -137,6 +141,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("folder", "folder", testBaseDrivePath, false, true, false), }, + inputFolderMap: map[string]string{}, scope: anyFolder, expect: assert.NoError, expectedCollectionPaths: []string{}, @@ -154,6 +159,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("package", "package", testBaseDrivePath, false, false, true), }, + inputFolderMap: map[string]string{}, scope: anyFolder, expect: assert.NoError, expectedCollectionPaths: []string{}, @@ -175,8 +181,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, @@ -215,8 +222,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, - scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], + expect: assert.NoError, expectedCollectionPaths: append( expectedPathAsSlice( suite.T(), @@ -257,12 +265,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false), driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false), - driveItem("folder", "folder", testBaseDrivePath+folderSub, false, true, false), + driveItem("folder2", "folder", testBaseDrivePath+folderSub, false, true, false), driveItem("package", "package", testBaseDrivePath, false, false, true), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, + inputFolderMap: map[string]string{}, scope: (&selectors.OneDriveBackup{}). Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0], expect: assert.NoError, @@ -276,7 +285,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedFileCount: 1, expectedContainerCount: 1, expectedMetadataPaths: map[string]string{ - "folder": expectedPathAsSlice( + "folder2": expectedPathAsSlice( suite.T(), tenant, user, @@ -295,8 +304,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), }, - scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, @@ -309,6 +319,34 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { // No child folders for subfolder so nothing here. expectedMetadataPaths: map[string]string{}, }, + { + testCase: "deleted folder and package", + items: []models.DriveItemable{ + delItem("folder", testBaseDrivePath, false, true, false), + delItem("package", testBaseDrivePath, false, false, true), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "package": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/package", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{}, + }, } for _, tt := range tests { @@ -316,7 +354,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { ctx, flush := tester.NewContext() defer flush() - paths := map[string]string{} + outputFolderMap := map[string]string{} + maps.Copy(outputFolderMap, tt.inputFolderMap) c := NewCollections( tenant, user, @@ -326,7 +365,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", "General", tt.items, paths) + err := c.UpdateCollections(ctx, "driveID", "General", tt.items, outputFolderMap) tt.expect(t, err) assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count") @@ -336,7 +375,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { assert.Contains(t, c.CollectionMap, collPath) } - assert.Equal(t, tt.expectedMetadataPaths, paths) + assert.Equal(t, tt.expectedMetadataPaths, outputFolderMap) }) } } @@ -361,3 +400,26 @@ func driveItem(id string, name string, path string, isFile, isFolder, isPackage return item } + +// delItem creates a DriveItemable that is marked as deleted. path must be set +// to the base drive path. +func delItem(id string, path string, isFile, isFolder, isPackage bool) models.DriveItemable { + item := models.NewDriveItem() + item.SetId(&id) + item.SetDeleted(models.NewDeleted()) + + parentReference := models.NewItemReference() + parentReference.SetPath(&path) + item.SetParentReference(parentReference) + + switch { + case isFile: + item.SetFile(models.NewFile()) + case isFolder: + item.SetFolder(models.NewFolder()) + case isPackage: + item.SetPackage(models.NewPackage_escaped()) + } + + return item +} From fd32770d2a96426fa5d912f95e8010bb79ab5cb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:28:05 +0000 Subject: [PATCH 02/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/sp?= =?UTF-8?q?f13/viper=20from=201.14.0=20to=201.15.0=20in=20/src=20(#2204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.14.0 to 1.15.0.
Release notes

Sourced from github.com/spf13/viper's releases.

v1.15.0

What's Changed

Exciting New Features 🎉

Enhancements 🚀

Breaking Changes đź› 

Dependency Updates ⬆️

New Contributors

Full Changelog: https://github.com/spf13/viper/compare/v1.14.0...v1.15.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/viper&package-manager=go_modules&previous-version=1.14.0&new-version=1.15.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 16 +++++++--------- src/go.sum | 30 ++++++++++++++---------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/go.mod b/src/go.mod index fe00918a9..1f3017de0 100644 --- a/src/go.mod +++ b/src/go.mod @@ -22,7 +22,7 @@ require ( github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.14.0 + github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 github.com/tidwall/pretty v1.2.1 github.com/tomlazar/table v0.1.2 @@ -40,19 +40,17 @@ require ( github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/microsoft/kiota-serialization-form-go v0.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.5 // indirect - github.com/spf13/afero v1.9.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.34.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( @@ -120,8 +118,8 @@ require ( golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect - google.golang.org/grpc v1.51.0 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/grpc v1.52.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/src/go.sum b/src/go.sum index d207d5c07..8650e7297 100644 --- a/src/go.sum +++ b/src/go.sum @@ -251,8 +251,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -303,10 +303,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= -github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -362,8 +360,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 h1:lQ3JvmcVO1/AMFbabvUSJ4YtJRpEAX9Qza73p5j03sw= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1/go.mod h1:4aKqcbhASNqjbrG0h9BmkzcWvPJGxbef4B+j0XfFrZo= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= @@ -372,8 +370,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -387,8 +385,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tg123/go-htpasswd v1.2.0 h1:UKp34m9H467/xklxUxU15wKRru7fwXoTojtxg25ITF0= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -747,8 +745,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc= -google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -765,8 +763,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 7f44db6c7c582290c832569606a3056457506b0d Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Fri, 20 Jan 2023 10:59:15 -0800 Subject: [PATCH 03/16] Fail operation if we failed to fetch item data (#2199) ## Description If GraphConnector reported errors while trying to fetch item data, fail the entire operation. This stops silent failures, but is a big hammer at the moment because there's no checks for items no longer being available. Error reporting is minimal as well, but some info should be in the log. **This effectively disables best-effort as any failure will result in the operation being reported as failed** ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * #2196 ## Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/operations/backup.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 89dbb340d..92a8b93c6 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -205,10 +205,21 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { return opStats.writeErr } + opStats.gc = gc.AwaitStatus() + + if opStats.gc.ErrorCount > 0 { + opStats.writeErr = multierror.Append(nil, opStats.writeErr, errors.Errorf( + "%v errors reported while fetching item data", + opStats.gc.ErrorCount, + )).ErrorOrNil() + + // Need to exit before we set started to true else we'll report no errors. + return opStats.writeErr + } + // should always be 1, since backups are 1:1 with resourceOwners. opStats.resourceCount = 1 opStats.started = true - opStats.gc = gc.AwaitStatus() return err } From bb7f54b049dfff6a0f7f401690a74c3755046e50 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Fri, 20 Jan 2023 11:18:59 -0800 Subject: [PATCH 04/16] Update OneDrive metadata with folder moves (#2193) ## Description Track folder moves and update the persisted metadata accordingly. Moves need to take into account subtree changes because OneDrive will not notify us of subfolders moving if nothing else was updated on the subfolder. Handle updates by making a copy of the folder map for ease of updating and later on ease of finding out which collection paths have changed ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * #2120 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/onedrive/collections.go | 47 ++++- .../connector/onedrive/collections_test.go | 199 +++++++++++++++++- src/internal/connector/onedrive/drive.go | 16 +- src/internal/connector/onedrive/item_test.go | 3 +- .../sharepoint/data_collections_test.go | 3 +- 5 files changed, 254 insertions(+), 14 deletions(-) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 73e1b1ba8..90734c43a 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -161,12 +161,16 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) { } // UpdateCollections initializes and adds the provided drive items to Collections -// A new collection is created for every drive folder (or package) +// A new collection is created for every drive folder (or package). +// oldPaths is the unchanged data that was loaded from the metadata file. +// newPaths starts as a copy of oldPaths and is updated as changes are found in +// the returned results. func (c *Collections) UpdateCollections( ctx context.Context, driveID, driveName string, items []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error { for _, item := range items { if item.GetRoot() != nil { @@ -201,7 +205,7 @@ func (c *Collections) UpdateCollections( // Nested folders also return deleted delta results so we don't have to // worry about doing a prefix search in the map to remove the subtree of // the deleted folder/package. - delete(paths, *item.GetId()) + delete(newPaths, *item.GetId()) // TODO(ashmrtn): Create a collection with state Deleted. @@ -217,13 +221,20 @@ func (c *Collections) UpdateCollections( return err } - // TODO(ashmrtn): Handle moves by setting the collection state if the - // collection doesn't already exist/have that state. - paths[*item.GetId()] = folderPath.String() + // Moved folders don't cause delta results for any subfolders nested in + // them. We need to go through and update paths to handle that. We only + // update newPaths so we don't accidentally clobber previous deletes. + // + // TODO(ashmrtn): Since we're also getting notifications about folder + // moves we may need to handle updates to a path of a collection we've + // already created and partially populated. + updatePath(newPaths, *item.GetId(), folderPath.String()) case item.GetFile() != nil: col, found := c.CollectionMap[collectionPath.String()] if !found { + // TODO(ashmrtn): Compare old and new path and set collection state + // accordingly. col = NewCollection( collectionPath, driveID, @@ -296,3 +307,27 @@ func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) boo return m.Matches(folderPathString) } + +func updatePath(paths map[string]string, id, newPath string) { + oldPath := paths[id] + if len(oldPath) == 0 { + paths[id] = newPath + return + } + + if oldPath == newPath { + return + } + + // We need to do a prefix search on the rest of the map to update the subtree. + // We don't need to make collections for all of these, as hierarchy merging in + // other components should take care of that. We do need to ensure that the + // resulting map contains all folders though so we know the next time around. + for folderID, p := range paths { + if !strings.HasPrefix(p, oldPath) { + continue + } + + paths[folderID] = strings.Replace(p, oldPath, newPath, 1) + } +} diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index a8520d52d..b9c313883 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -319,6 +319,168 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { // No child folders for subfolder so nothing here. expectedMetadataPaths: map[string]string{}, }, + { + testCase: "not moved folder tree", + items: []models.DriveItemable{ + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + }, + { + testCase: "moved folder tree", + items: []models.DriveItemable{ + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + }, + { + testCase: "moved folder tree and subfolder 1", + items: []models.DriveItemable{ + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/subfolder", + )[0], + }, + }, + { + testCase: "moved folder tree and subfolder 2", + items: []models.DriveItemable{ + driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), + driveItem("folder", "folder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/a-folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/subfolder", + )[0], + }, + }, { testCase: "deleted folder and package", items: []models.DriveItemable{ @@ -347,6 +509,41 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { expectedContainerCount: 0, expectedMetadataPaths: map[string]string{}, }, + { + testCase: "delete folder tree move subfolder", + items: []models.DriveItemable{ + delItem("folder", testBaseDrivePath, false, true, false), + driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), + }, + inputFolderMap: map[string]string{ + "folder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + )[0], + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder/subfolder", + )[0], + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: []string{}, + expectedItemCount: 0, + expectedFileCount: 0, + expectedContainerCount: 0, + expectedMetadataPaths: map[string]string{ + "subfolder": expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/subfolder", + )[0], + }, + }, } for _, tt := range tests { @@ -365,7 +562,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", "General", tt.items, outputFolderMap) + err := c.UpdateCollections(ctx, "driveID", "General", tt.items, tt.inputFolderMap, outputFolderMap) tt.expect(t, err) assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count") diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 18a1b2477..17ee6f96d 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -13,6 +13,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" "github.com/microsoftgraph/msgraph-sdk-go/sites" "github.com/pkg/errors" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" @@ -167,7 +168,8 @@ type itemCollector func( ctx context.Context, driveID, driveName string, driveItems []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error // collectItems will enumerate all items in the specified drive and hand them to the @@ -182,9 +184,12 @@ func collectItems( newDeltaURL = "" // TODO(ashmrtn): Eventually this should probably be a parameter so we can // take in previous paths. - paths = map[string]string{} + oldPaths = map[string]string{} + newPaths = map[string]string{} ) + maps.Copy(newPaths, oldPaths) + // TODO: Specify a timestamp in the delta query // https://docs.microsoft.com/en-us/graph/api/driveitem-delta? // view=graph-rest-1.0&tabs=http#example-4-retrieving-delta-results-using-a-timestamp @@ -221,7 +226,7 @@ func collectItems( ) } - err = collector(ctx, driveID, driveName, r.GetValue(), paths) + err = collector(ctx, driveID, driveName, r.GetValue(), oldPaths, newPaths) if err != nil { return "", nil, err } @@ -240,7 +245,7 @@ func collectItems( builder = msdrives.NewItemRootDeltaRequestBuilder(*nextLink, service.Adapter()) } - return newDeltaURL, paths, nil + return newDeltaURL, newPaths, nil } // getFolder will lookup the specified folder name under `parentFolderID` @@ -356,7 +361,8 @@ func GetAllFolders( innerCtx context.Context, driveID, driveName string, items []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error { for _, item := range items { // Skip the root item. diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 5c2e8c335..f53607328 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -99,7 +99,8 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { ctx context.Context, driveID, driveName string, items []models.DriveItemable, - paths map[string]string, + oldPaths map[string]string, + newPaths map[string]string, ) error { for _, item := range items { if item.GetFile() != nil { diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index 2934b2fa1..9fc538e2c 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -88,6 +88,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() { defer flush() paths := map[string]string{} + newPaths := map[string]string{} c := onedrive.NewCollections( tenant, site, @@ -96,7 +97,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() { &MockGraphService{}, nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths) + err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths, newPaths) test.expect(t, err) assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") From b3d4b4687b4bbe187be236631a130c01d924236b Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 20 Jan 2023 13:09:59 -0700 Subject: [PATCH 05/16] discovery api, filter guest and external users (#2188) ## Description Adds the api client pkg pattern to the connector/ discovery package. Most code changes are plain lift-n-shift, with minor clean-ups along the way. User retrieval is now filtered to only include member and on-premise accounts. ## Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included ## Type of change - [x] :sunflower: Feature ## Issue(s) * #2094 ## Test Plan - [x] :muscle: Manual - [x] :zap: Unit test --- CHANGELOG.md | 2 + src/internal/connector/data_collections.go | 8 +- src/internal/connector/discovery/api/api.go | 57 ++++++ src/internal/connector/discovery/api/users.go | 166 ++++++++++++++++++ .../{discovery_test.go => api/users_test.go} | 12 +- src/internal/connector/discovery/discovery.go | 137 ++++----------- src/internal/connector/graph_connector.go | 14 +- .../connector/graph_connector_test.go | 5 +- src/pkg/services/m365/m365.go | 2 +- 9 files changed, 281 insertions(+), 122 deletions(-) create mode 100644 src/internal/connector/discovery/api/api.go create mode 100644 src/internal/connector/discovery/api/users.go rename src/internal/connector/discovery/{discovery_test.go => api/users_test.go} (84%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07c061f12..248260c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Check if the user specified for an exchange backup operation has a mailbox. - Handle case where user's drive has not been initialized - Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163)) +- Guest and External users (for cloud accounts) and non-on-premise users (for systems that use on-prem AD syncs) are now excluded from backup and restore operations. + ## [v0.1.0] (alpha) - 2023-01-13 diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index 2ba2e59a4..7eaf1c517 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -8,8 +8,8 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/discovery" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/exchange" - "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/support" @@ -44,7 +44,7 @@ func (gc *GraphConnector) DataCollections( return nil, err } - serviceEnabled, err := checkServiceEnabled(ctx, gc.Service, path.ServiceType(sels.Service), sels.DiscreteOwner) + serviceEnabled, err := checkServiceEnabled(ctx, gc.Owners.Users(), path.ServiceType(sels.Service), sels.DiscreteOwner) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) erro func checkServiceEnabled( ctx context.Context, - gs graph.Servicer, + au api.Users, service path.ServiceType, resource string, ) (bool, error) { @@ -147,7 +147,7 @@ func checkServiceEnabled( return true, nil } - _, info, err := discovery.User(ctx, gs, resource) + _, info, err := discovery.User(ctx, au, resource) if err != nil { return false, err } diff --git a/src/internal/connector/discovery/api/api.go b/src/internal/connector/discovery/api/api.go new file mode 100644 index 000000000..20fc1b25d --- /dev/null +++ b/src/internal/connector/discovery/api/api.go @@ -0,0 +1,57 @@ +package api + +import ( + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/account" +) + +// --------------------------------------------------------------------------- +// interfaces +// --------------------------------------------------------------------------- + +// Client is used to fulfill the interface for discovery +// queries that are traditionally backed by GraphAPI. A +// struct is used in this case, instead of deferring to +// pure function wrappers, so that the boundary separates the +// granular implementation of the graphAPI and kiota away +// from the exchange package's broader intents. +type Client struct { + Credentials account.M365Config + + // The stable service is re-usable for any non-paged request. + // This allows us to maintain performance across async requests. + stable graph.Servicer +} + +// NewClient produces a new exchange api client. Must be used in +// place of creating an ad-hoc client struct. +func NewClient(creds account.M365Config) (Client, error) { + s, err := newService(creds) + if err != nil { + return Client{}, err + } + + return Client{creds, s}, nil +} + +// service generates a new service. Used for paged and other long-running +// requests instead of the client's stable service, so that in-flight state +// within the adapter doesn't get clobbered +func (c Client) service() (*graph.Service, error) { + return newService(c.Credentials) +} + +func newService(creds account.M365Config) (*graph.Service, error) { + adapter, err := graph.CreateAdapter( + creds.AzureTenantID, + creds.AzureClientID, + creds.AzureClientSecret, + ) + if err != nil { + return nil, errors.Wrap(err, "generating graph api service client") + } + + return graph.NewService(adapter), nil +} diff --git a/src/internal/connector/discovery/api/users.go b/src/internal/connector/discovery/api/users.go new file mode 100644 index 000000000..ff41ee06f --- /dev/null +++ b/src/internal/connector/discovery/api/users.go @@ -0,0 +1,166 @@ +package api + +import ( + "context" + + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Users() Users { + return Users{c} +} + +// Users is an interface-compliant provider of the client. +type Users struct { + Client +} + +// --------------------------------------------------------------------------- +// structs +// --------------------------------------------------------------------------- + +type UserInfo struct { + DiscoveredServices map[path.ServiceType]struct{} +} + +func newUserInfo() *UserInfo { + return &UserInfo{ + DiscoveredServices: map[path.ServiceType]struct{}{ + path.ExchangeService: {}, + path.OneDriveService: {}, + }, + } +} + +// --------------------------------------------------------------------------- +// methods +// --------------------------------------------------------------------------- + +const ( + userSelectID = "id" + userSelectPrincipalName = "userPrincipalName" + userSelectDisplayName = "displayName" +) + +// Filter out both guest users, and (for on-prem installations) non-synced users. +// The latter filter makes an assumption that no on-prem users are guests; this might +// require more fine-tuned controls in the future. +// https://stackoverflow.com/questions/64044266/error-message-unsupported-or-invalid-query-filter-clause-specified-for-property +// +//nolint:lll +var userFilterNoGuests = "onPremisesSyncEnabled eq true OR userType eq 'Member'" + +func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration { + return &users.UsersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.UsersRequestBuilderGetQueryParameters{ + Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName}, + Filter: fs, + }, + } +} + +// GetAll retrieves all users. +func (c Users) GetAll(ctx context.Context) ([]models.Userable, error) { + service, err := c.service() + if err != nil { + return nil, err + } + + resp, err := service.Client().Users().Get(ctx, userOptions(&userFilterNoGuests)) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "getting all users") + } + + iter, err := msgraphgocore.NewPageIterator( + resp, + service.Adapter(), + models.CreateUserCollectionResponseFromDiscriminatorValue) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "constructing user iterator") + } + + var ( + iterErrs error + us = make([]models.Userable, 0) + ) + + iterator := func(item any) bool { + u, err := validateUser(item) + if err != nil { + iterErrs = support.WrapAndAppend("validating user", err, iterErrs) + } else { + us = append(us, u) + } + + return true + } + + if err := iter.Iterate(ctx, iterator); err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "iterating all users") + } + + return us, iterErrs +} + +func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, error) { + user, err := c.stable.Client().UsersById(userID).Get(ctx, nil) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "getting user by id") + } + + return user, nil +} + +func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) { + // Assume all services are enabled + // then filter down to only services the user has enabled + userInfo := newUserInfo() + + // TODO: OneDrive + + _, err := c.stable.Client().UsersById(userID).MailFolders().Get(ctx, nil) + if err != nil { + if !graph.IsErrExchangeMailFolderNotFound(err) { + return nil, support.ConnectorStackErrorTraceWrap(err, "getting user's exchange mailfolders") + } + + delete(userInfo.DiscoveredServices, path.ExchangeService) + } + + return userInfo, nil +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// validateUser ensures the item is a Userable, and contains the necessary +// identifiers that we handle with all users. +// returns the item as a Userable model. +func validateUser(item any) (models.Userable, error) { + m, ok := item.(models.Userable) + if !ok { + return nil, errors.Errorf("expected Userable, got %T", item) + } + + if m.GetId() == nil { + return nil, errors.Errorf("missing ID") + } + + if m.GetUserPrincipalName() == nil { + return nil, errors.New("missing principalName") + } + + return m, nil +} diff --git a/src/internal/connector/discovery/discovery_test.go b/src/internal/connector/discovery/api/users_test.go similarity index 84% rename from src/internal/connector/discovery/discovery_test.go rename to src/internal/connector/discovery/api/users_test.go index e9349ba55..160806cf6 100644 --- a/src/internal/connector/discovery/discovery_test.go +++ b/src/internal/connector/discovery/api/users_test.go @@ -1,4 +1,4 @@ -package discovery +package api import ( "reflect" @@ -8,15 +8,15 @@ import ( "github.com/stretchr/testify/suite" ) -type DiscoverySuite struct { +type UsersUnitSuite struct { suite.Suite } -func TestDiscoverySuite(t *testing.T) { - suite.Run(t, new(DiscoverySuite)) +func TestUsersUnitSuite(t *testing.T) { + suite.Run(t, new(UsersUnitSuite)) } -func (suite *DiscoverySuite) TestParseUser() { +func (suite *UsersUnitSuite) TestValidateUser() { t := suite.T() name := "testuser" @@ -60,7 +60,7 @@ func (suite *DiscoverySuite) TestParseUser() { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseUser(tt.args) + got, err := validateUser(tt.args) if (err != nil) != tt.wantErr { t.Errorf("parseUser() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/src/internal/connector/discovery/discovery.go b/src/internal/connector/discovery/discovery.go index a9f2266d1..3f75f74c4 100644 --- a/src/internal/connector/discovery/discovery.go +++ b/src/internal/connector/discovery/discovery.go @@ -3,125 +3,52 @@ package discovery import ( "context" - msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" - msuser "github.com/microsoftgraph/msgraph-sdk-go/users" "github.com/pkg/errors" - "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/support" - "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/internal/connector/discovery/api" ) -const ( - userSelectID = "id" - userSelectPrincipalName = "userPrincipalName" - userSelectDisplayName = "displayName" -) +// --------------------------------------------------------------------------- +// interfaces +// --------------------------------------------------------------------------- -func Users(ctx context.Context, gs graph.Servicer, tenantID string) ([]models.Userable, error) { - users := make([]models.Userable, 0) - - options := &msuser.UsersRequestBuilderGetRequestConfiguration{ - QueryParameters: &msuser.UsersRequestBuilderGetQueryParameters{ - Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName}, - }, - } - - response, err := gs.Client().Users().Get(ctx, options) - if err != nil { - return nil, errors.Wrapf( - err, - "retrieving resources for tenant %s: %s", - tenantID, - support.ConnectorStackErrorTrace(err), - ) - } - - iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(), - models.CreateUserCollectionResponseFromDiscriminatorValue) - if err != nil { - return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) - } - - var iterErrs error - - callbackFunc := func(item interface{}) bool { - u, err := parseUser(item) - if err != nil { - iterErrs = support.WrapAndAppend("discovering users: ", err, iterErrs) - return true - } - - users = append(users, u) - - return true - } - - if err := iter.Iterate(ctx, callbackFunc); err != nil { - return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) - } - - return users, iterErrs +type getAller interface { + GetAll(context.Context) ([]models.Userable, error) } -type UserInfo struct { - DiscoveredServices map[path.ServiceType]struct{} +type getter interface { + GetByID(context.Context, string) (models.Userable, error) } -func User(ctx context.Context, gs graph.Servicer, userID string) (models.Userable, *UserInfo, error) { - user, err := gs.Client().UsersById(userID).Get(ctx, nil) +type getInfoer interface { + GetInfo(context.Context, string) (*api.UserInfo, error) +} + +type getWithInfoer interface { + getter + getInfoer +} + +// --------------------------------------------------------------------------- +// api +// --------------------------------------------------------------------------- + +// Users fetches all users in the tenant. +func Users(ctx context.Context, ga getAller) ([]models.Userable, error) { + return ga.GetAll(ctx) +} + +func User(ctx context.Context, gwi getWithInfoer, userID string) (models.Userable, *api.UserInfo, error) { + u, err := gwi.GetByID(ctx, userID) if err != nil { - return nil, nil, errors.Wrapf( - err, - "retrieving resource for tenant: %s", - support.ConnectorStackErrorTrace(err), - ) + return nil, nil, errors.Wrap(err, "getting user") } - // Assume all services are enabled - userInfo := &UserInfo{ - DiscoveredServices: map[path.ServiceType]struct{}{ - path.ExchangeService: {}, - path.OneDriveService: {}, - }, - } - - // Discover which services the user has enabled - - // Exchange: Query `MailFolders` - _, err = gs.Client().UsersById(userID).MailFolders().Get(ctx, nil) + ui, err := gwi.GetInfo(ctx, userID) if err != nil { - if !graph.IsErrExchangeMailFolderNotFound(err) { - return nil, nil, errors.Wrapf( - err, - "retrieving mail folders for tenant: %s", - support.ConnectorStackErrorTrace(err), - ) - } - - delete(userInfo.DiscoveredServices, path.ExchangeService) + return nil, nil, errors.Wrap(err, "getting user info") } - // TODO: OneDrive - - return user, userInfo, nil -} - -// parseUser extracts information from `models.Userable` we care about -func parseUser(item interface{}) (models.Userable, error) { - m, ok := item.(models.Userable) - if !ok { - return nil, errors.New("iteration retrieved non-User item") - } - - if m.GetId() == nil { - return nil, errors.Errorf("no ID for User") - } - - if m.GetUserPrincipalName() == nil { - return nil, errors.Errorf("no principal name for User: %s", *m.GetId()) - } - - return m, nil + return u, ui, nil } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index e047a017b..68f03d048 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -15,6 +15,7 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/connector/discovery" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" @@ -37,7 +38,9 @@ import ( // GraphRequestAdapter from the msgraph-sdk-go. Additional fields are for // bookkeeping and interfacing with other component. type GraphConnector struct { - Service graph.Servicer + Service graph.Servicer + Owners api.Client + tenant string Users map[string]string // key value Sites map[string]string // key value @@ -74,12 +77,15 @@ func NewGraphConnector(ctx context.Context, acct account.Account, r resource) (* credentials: m365, } - gService, err := gc.createService() + gc.Service, err = gc.createService() if err != nil { return nil, errors.Wrap(err, "creating service connection") } - gc.Service = gService + gc.Owners, err = api.NewClient(m365) + if err != nil { + return nil, errors.Wrap(err, "creating api client") + } // TODO(ashmrtn): When selectors only encapsulate a single resource owner that // is not a wildcard don't populate users or sites when making the connector. @@ -121,7 +127,7 @@ func (gc *GraphConnector) setTenantUsers(ctx context.Context) error { ctx, end := D.Span(ctx, "gc:setTenantUsers") defer end() - users, err := discovery.Users(ctx, gc.Service, gc.tenant) + users, err := discovery.Users(ctx, gc.Owners.Users()) if err != nil { return err } diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index d635ee6f9..71eff095a 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/suite" "golang.org/x/exp/maps" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/support" @@ -174,10 +175,10 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { ctx, flush := tester.NewContext() defer flush() - service, err := newConnector.createService() + owners, err := api.NewClient(suite.connector.credentials) require.NoError(suite.T(), err) - newConnector.Service = service + newConnector.Owners = owners suite.Empty(len(newConnector.Users)) err = newConnector.setTenantUsers(ctx) diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 19da0a21f..ba9cc2a59 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -25,7 +25,7 @@ func Users(ctx context.Context, m365Account account.Account) ([]*User, error) { return nil, errors.Wrap(err, "could not initialize M365 graph connection") } - users, err := discovery.Users(ctx, gc.Service, m365Account.ID()) + users, err := discovery.Users(ctx, gc.Owners.Users()) if err != nil { return nil, err } From 7872201f73958d98129adfd1132fe6631c03016b Mon Sep 17 00:00:00 2001 From: Niraj Tolia Date: Fri, 20 Jan 2023 12:13:25 -0800 Subject: [PATCH 06/16] Move to wow.js fork (#2210) ## Description wow.js got relicensed. Switch to the MIT fork (the old license) ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :bug: Bugfix ## Test Plan - [x] :muscle: Manual --- website/package-lock.json | 25 +++++++++------------- website/package.json | 2 +- website/src/components/parts/KeyLoveFAQ.js | 4 ++-- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index ef0577fb2..c781b7458 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -26,7 +26,7 @@ "react-dom": "^17.0.2", "sass": "^1.57.1", "tw-elements": "^1.0.0-alpha13", - "wowjs": "^1.1.3" + "wow.js": "^1.2.2" }, "devDependencies": { "@docusaurus/module-type-aliases": "2.2.0", @@ -14543,13 +14543,11 @@ "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", "license": "MIT" }, - "node_modules/wowjs": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz", - "integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==", - "dependencies": { - "animate.css": "latest" - } + "node_modules/wow.js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz", + "integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A==", + "deprecated": "deprecated in favour of aos (Animate On Scroll)" }, "node_modules/wrap-ansi": { "version": "8.0.1", @@ -24231,13 +24229,10 @@ "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==" }, - "wowjs": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wowjs/-/wowjs-1.1.3.tgz", - "integrity": "sha512-HQp1gi56wYmjOYYOMZ08TnDGpT+AO21RJVa0t1NJ3jU8l3dMyP+sY7TO/lilzVp4JFjW88bBY87RnpxdpSKofA==", - "requires": { - "animate.css": "latest" - } + "wow.js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/wow.js/-/wow.js-1.2.2.tgz", + "integrity": "sha512-YTW9eiZimHCJDWofsiz2507txaPteUiQD461I/D8533AiRAn3+Y68/1LDuQ3OTgPjagGZLPYKrpoSgjzeQrO6A==" }, "wrap-ansi": { "version": "8.0.1", diff --git a/website/package.json b/website/package.json index 6c27fe784..0cb897b46 100644 --- a/website/package.json +++ b/website/package.json @@ -32,7 +32,7 @@ "react-dom": "^17.0.2", "sass": "^1.57.1", "tw-elements": "^1.0.0-alpha13", - "wowjs": "^1.1.3" + "wow.js": "^1.2.2" }, "devDependencies": { "@docusaurus/module-type-aliases": "2.2.0", diff --git a/website/src/components/parts/KeyLoveFAQ.js b/website/src/components/parts/KeyLoveFAQ.js index c07c35aba..fb13be58d 100644 --- a/website/src/components/parts/KeyLoveFAQ.js +++ b/website/src/components/parts/KeyLoveFAQ.js @@ -5,12 +5,12 @@ export default function KeyLoveFAQ() { const jarallaxRef = useRef(null); useEffect(() => { if (typeof window !== "undefined") { - const WOW = require("wowjs"); + const WOW = require("wow.js"); const father = require("feather-icons"); const jarallax = require("jarallax"); require("tw-elements"); - new WOW.WOW({ + new WOW({ live: false, }).init(); father.replace(); From cd77f43fb2d6aff0e355664928b6d96018a4d427 Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 20 Jan 2023 16:01:20 -0500 Subject: [PATCH 07/16] CLI: Updates for Exchange Help (#2191) ## Description Removes references to M365 ID as instruction. ## Does this PR need a docs update or release note? - [x] :no_entry: No, self-documenting ## Type of change - [x] :bug: Bugfix ## Issue(s) * closes #2190 ## Test Plan - [x] :zap: Unit test --- src/cli/backup/exchange.go | 8 ++++---- src/cli/backup/onedrive.go | 4 ++-- src/cli/restore/exchange.go | 2 +- src/cli/restore/onedrive.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 63a1f6393..ac6455522 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -61,7 +61,7 @@ const ( const ( exchangeServiceCommand = "exchange" - exchangeServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" + exchangeServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" exchangeServiceCommandDeleteUseSuffix = "--backup " exchangeServiceCommandDetailsUseSuffix = "--backup " ) @@ -115,7 +115,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar( &user, utils.UserFN, nil, - "Backup Exchange data by user ID; accepts '"+utils.Wildcard+"' to select all users") + "Backup Exchange data by a user's email; accepts '"+utils.Wildcard+"' to select all users") fs.StringSliceVar( &exchangeData, utils.DataFN, nil, @@ -274,7 +274,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { users, err := m365.UserPNs(ctx, acct) if err != nil { - return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users")) + return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 user(s)")) } var ( @@ -347,7 +347,7 @@ func exchangeBackupCreateSelectors(userIDs, data []string) *selectors.ExchangeBa func validateExchangeBackupCreateFlags(userIDs, data []string) error { if len(userIDs) == 0 { - return errors.New("--user requires one or more ids or the wildcard *") + return errors.New("--user requires one or more email addresses or the wildcard '*'") } for _, d := range data { diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index 3454a9b3b..60a055dce 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -29,7 +29,7 @@ import ( const ( oneDriveServiceCommand = "onedrive" - oneDriveServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" + oneDriveServiceCommandCreateUseSuffix = "--user | '" + utils.Wildcard + "'" oneDriveServiceCommandDeleteUseSuffix = "--backup " oneDriveServiceCommandDetailsUseSuffix = "--backup " ) @@ -85,7 +85,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar(&user, utils.UserFN, nil, - "Backup OneDrive data by user ID; accepts '"+utils.Wildcard+"' to select all users. (required)") + "Backup OneDrive data by user's email address; accepts '"+utils.Wildcard+"' to select all users. (required)") options.AddOperationFlags(c) case listCommand: diff --git a/src/cli/restore/exchange.go b/src/cli/restore/exchange.go index 6d67fa03e..d0801061f 100644 --- a/src/cli/restore/exchange.go +++ b/src/cli/restore/exchange.go @@ -63,7 +63,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar(&user, utils.UserFN, nil, - "Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.") + "Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.") // email flags fs.StringSliceVar(&email, diff --git a/src/cli/restore/onedrive.go b/src/cli/restore/onedrive.go index 0932461a3..526db414b 100644 --- a/src/cli/restore/onedrive.go +++ b/src/cli/restore/onedrive.go @@ -49,7 +49,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { fs.StringSliceVar(&user, utils.UserFN, nil, - "Restore data by user ID; accepts '"+utils.Wildcard+"' to select all users.") + "Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.") // onedrive hierarchy (path/name) flags From ea2d9cecebe88fa88730f783cf632f22df42254d Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Fri, 20 Jan 2023 13:40:48 -0800 Subject: [PATCH 08/16] Report errors when building details (#2206) ## Description Report an error if an item in details that was sourced from a base snapshot doesn't have a previous path. Items that don't have a previous path cannot be sourced from a base snapshot's backup details meaning the item will be stored but will not be searchable/restorable by users Logging was not used because no context is available in the kopia callback that is checking if the previous path is nil ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * closes #1915 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 8 +++++++- src/internal/kopia/upload_test.go | 34 ++++++++++++++++++++++++++++++- src/internal/kopia/wrapper.go | 5 +++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index d9320f2a6..52452ffa9 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -134,6 +134,7 @@ type corsoProgress struct { toMerge map[string]path.Path mu sync.RWMutex totalBytes int64 + errs *multierror.Error } // Kopia interface function used as a callback when kopia finishes processing a @@ -162,8 +163,13 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) { // These items were sourced from a base snapshot or were cached in kopia so we // never had to materialize their details in-memory. if d.info == nil { - // TODO(ashmrtn): We should probably be returning an error here? if d.prevPath == nil { + cp.errs = multierror.Append(cp.errs, errors.Errorf( + "item sourced from previous backup with no previous path. Service: %s, Category: %s", + d.repoPath.Service().String(), + d.repoPath.Category().String(), + )) + return } diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index a3a865cd8..ff86db13b 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -468,7 +468,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { for k, v := range ci { if cachedTest.cached { - cp.CachedFile(k, 42) + cp.CachedFile(k, v.totalBytes) } cp.FinishedFile(k, v.err) @@ -489,6 +489,38 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() { } } +func (suite *CorsoProgressUnitSuite) TestFinishedFileCachedNoPrevPathErrors() { + t := suite.T() + bd := &details.Builder{} + cachedItems := map[string]testInfo{ + suite.targetFileName: { + info: &itemDetails{info: nil, repoPath: suite.targetFilePath}, + err: nil, + totalBytes: 100, + }, + } + cp := corsoProgress{ + UploadProgress: &snapshotfs.NullUploadProgress{}, + deets: bd, + pending: map[string]*itemDetails{}, + } + + for k, v := range cachedItems { + cp.put(k, v.info) + } + + require.Len(t, cp.pending, len(cachedItems)) + + for k, v := range cachedItems { + cp.CachedFile(k, v.totalBytes) + cp.FinishedFile(k, v.err) + } + + assert.Empty(t, cp.pending) + assert.Empty(t, bd.Details().Entries) + assert.Error(t, cp.errs.ErrorOrNil()) +} + func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchyNewItem() { t := suite.T() // Order of folders in hierarchy from root to leaf (excluding the item). diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index feab0e687..8171c853a 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -160,10 +160,11 @@ func (w Wrapper) BackupCollections( progress, ) if err != nil { - return nil, nil, nil, err + combinedErrs := multierror.Append(nil, err, progress.errs) + return nil, nil, nil, combinedErrs.ErrorOrNil() } - return s, progress.deets, progress.toMerge, nil + return s, progress.deets, progress.toMerge, progress.errs.ErrorOrNil() } func (w Wrapper) makeSnapshotWithRoot( From 6d615810cdaf9b5745d8b88687a1e9eb6932db48 Mon Sep 17 00:00:00 2001 From: Danny Date: Sat, 21 Jan 2023 00:50:23 -0500 Subject: [PATCH 09/16] CLI: Backup: SharePoint: Pages Enable (#2213) ## Description Adds the functionality to select SharePoint Pages for backup. ## Does this PR need a docs update or release note? SharePoint commands remain hidden in terms of pages. - [x] :clock1: Yes, but in a later PR ## Type of change - [x] :sunflower: Feature ## Issue(s) * closes #2216 * closes #2107 ## Test Plan - [x] :zap: Unit test --- src/cli/backup/sharepoint.go | 53 +++++++++++++++++++++++++------ src/cli/backup/sharepoint_test.go | 17 +++++++--- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index d5624380f..4eb62dca0 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -27,9 +27,12 @@ import ( // setup and globals // ------------------------------------------------------------------------------------------------ +// sharePoint bucket info from flags var ( libraryItems []string libraryPaths []string + pageFolders []string + page []string site []string weburl []string @@ -38,6 +41,7 @@ var ( const ( dataLibraries = "libraries" + dataPages = "pages" ) const ( @@ -89,11 +93,10 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { utils.WebURLFN, nil, "Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") - // TODO: implement fs.StringSliceVar( &sharepointData, utils.DataFN, nil, - "Select one or more types of data to backup: "+dataLibraries+".") + "Select one or more types of data to backup: "+dataLibraries+" or "+dataPages+".") options.AddOperationFlags(c) case listCommand: @@ -128,11 +131,22 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { fs.StringArrayVar(&site, utils.SiteFN, nil, - "Backup SharePoint data by site ID; accepts '"+utils.Wildcard+"' to select all sites.") + "Select backup details by site ID; accepts '"+utils.Wildcard+"' to select all sites.") fs.StringSliceVar(&weburl, utils.WebURLFN, nil, - "Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") + "Select backup data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") + + fs.StringSliceVar( + &pageFolders, + utils.PageFN, nil, + "Select backup data by site ID; accepts '"+utils.Wildcard+"' to select all sites.") + + fs.StringSliceVar( + &page, + utils.PageItemFN, nil, + "Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.", + ) // info flags @@ -179,7 +193,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return nil } - if err := validateSharePointBackupCreateFlags(site, weburl); err != nil { + if err := validateSharePointBackupCreateFlags(site, weburl, sharepointData); err != nil { return err } @@ -200,7 +214,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs")) } - sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, gc) + sel, err := sharePointBackupCreateSelectors(ctx, site, weburl, sharepointData, gc) if err != nil { return Only(ctx, errors.Wrap(err, "Retrieving up sharepoint sites by ID and WebURL")) } @@ -250,7 +264,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return nil } -func validateSharePointBackupCreateFlags(sites, weburls []string) error { +func validateSharePointBackupCreateFlags(sites, weburls, data []string) error { if len(sites) == 0 && len(weburls) == 0 { return errors.New( "requires one or more --" + @@ -260,13 +274,21 @@ func validateSharePointBackupCreateFlags(sites, weburls []string) error { ) } + for _, d := range data { + if d != dataLibraries && d != dataPages { + return errors.New( + d + " is an unrecognized data type; either " + dataLibraries + "or " + dataPages, + ) + } + } + return nil } // TODO: users might specify a data type, this only supports AllData(). func sharePointBackupCreateSelectors( ctx context.Context, - sites, weburls []string, + sites, weburls, data []string, gc *connector.GraphConnector, ) (*selectors.SharePointBackup, error) { if len(sites) == 0 && len(weburls) == 0 { @@ -297,7 +319,20 @@ func sharePointBackupCreateSelectors( } sel := selectors.NewSharePointBackup(union) - sel.Include(sel.AllData()) + if len(data) == 0 { + sel.Include(sel.AllData()) + + return sel, nil + } + + for _, d := range data { + switch d { + case dataLibraries: + sel.Include(sel.Libraries(selectors.Any())) + case dataPages: + sel.Include(sel.Pages(selectors.Any())) + } + } return sel, nil } diff --git a/src/cli/backup/sharepoint_test.go b/src/cli/backup/sharepoint_test.go index d568d4cc6..89e40a9f3 100644 --- a/src/cli/backup/sharepoint_test.go +++ b/src/cli/backup/sharepoint_test.go @@ -98,12 +98,13 @@ func (suite *SharePointSuite) TestValidateSharePointBackupCreateFlags() { } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl)) + test.expect(t, validateSharePointBackupCreateFlags(test.site, test.weburl, nil)) }) } } func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { + comboString := []string{"id_1", "id_2"} gc := &connector.GraphConnector{ Sites: map[string]string{ "url_1": "id_1", @@ -115,6 +116,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { name string site []string weburl []string + data []string expect []string expectScopesLen int }{ @@ -163,7 +165,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { name: "duplicate sites and urls", site: []string{"id_1", "id_2"}, weburl: []string{"url_1", "url_2"}, - expect: []string{"id_1", "id_2"}, + expect: comboString, expectScopesLen: 2, }, { @@ -175,18 +177,25 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() { }, { name: "unnecessary url wildcard", - site: []string{"id_1", "id_2"}, + site: comboString, weburl: []string{"url_1", utils.Wildcard}, expect: selectors.Any(), expectScopesLen: 2, }, + { + name: "Pages", + site: comboString, + data: []string{dataPages}, + expect: comboString, + expectScopesLen: 1, + }, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { ctx, flush := tester.NewContext() defer flush() - sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, gc) + sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, test.data, gc) require.NoError(t, err) assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners()) From 6f98cbf33ad43b0cb6d940c05d714d392951cbe5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 06:22:00 +0000 Subject: [PATCH 10/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.183=20to=201.44.184=20in=20/src=20(#?= =?UTF-8?q?2229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.183 to 1.44.184.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.184 (2023-01-20)

Service Client Updates

  • service/ec2: Updates service API
    • C6in, M6in, M6idn, R6in and R6idn instances are powered by 3rd Generation Intel Xeon Scalable processors (code named Ice Lake) with an all-core turbo frequency of 3.5 GHz.
  • service/ivs: Updates service API and documentation
  • service/quicksight: Updates service API and documentation
    • This release adds support for data bars in QuickSight table and increases pivot table field well limit.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.183&new-version=1.44.184)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 1f3017de0..b46328257 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2023011220 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.183 + github.com/aws/aws-sdk-go v1.44.184 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/src/go.sum b/src/go.sum index 8650e7297..ea0ca662c 100644 --- a/src/go.sum +++ b/src/go.sum @@ -62,8 +62,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk= -github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo= +github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 66d17c5fd53e5f1414840fc5e3e39c754a7f19ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C4=8Dnica=20Mellifera?= Date: Mon, 23 Jan 2023 08:02:55 -0800 Subject: [PATCH 11/16] link directly to getting started directions (#2232) ## Description rather than link to the top level of the docs, this now goes straight to the quick-start. ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [x] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * # ## Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cdb00ddf3..33341e4e7 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ services, possibly beyond M365, will expand based on the interest and needs of t # Getting Started -See the [Corso Documentation](https://corsobackup.io/docs/intro) for more information. +See the [Corso Quickstart](https://corsobackup.io/docs/quickstart/) on our docs page. # Building Corso From b6a7095c7f5c8ac6f9a538b217051a1f955ddf22 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Mon, 23 Jan 2023 10:35:16 -0800 Subject: [PATCH 12/16] Don't retry if an item was not found (#2233) ## Description When fetching item data, don't backoff and retry if Graph reported the item was not found. In this case, we mark it as succeeded as we can never get the data anyway. ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup ## Issue(s) * closes #2217 ## Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../exchange/exchange_data_collection.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index e168ed082..ce37beab6 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -257,6 +257,24 @@ func (col *Collection) streamItems(ctx context.Context) { break } + // If the data is no longer available just return here and chalk it up + // as a success. There's no reason to retry and no way we can backup up + // enough information to restore the item anyway. + if e := graph.IsErrDeletedInFlight(err); e != nil { + atomic.AddInt64(&success, 1) + logger.Ctx(ctx).Infow( + "Graph reported item not found", + "error", + e, + "service", + path.ExchangeService.String(), + "category", + col.category.String, + ) + + return + } + if i < numberOfRetries { time.Sleep(time.Duration(3*(i+1)) * time.Second) } @@ -270,6 +288,16 @@ func (col *Collection) streamItems(ctx context.Context) { // attempted items. if e := graph.IsErrDeletedInFlight(err); e != nil { atomic.AddInt64(&success, 1) + logger.Ctx(ctx).Infow( + "Graph reported item not found", + "error", + e, + "service", + path.ExchangeService.String(), + "category", + col.category.String, + ) + return } From e43455cc49a2684c433944a90000b61ed9a2b68b Mon Sep 17 00:00:00 2001 From: Keepers Date: Mon, 23 Jan 2023 12:52:26 -0700 Subject: [PATCH 13/16] re-enable stderr logs in tests (#2215) ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :bug: Bugfix - [x] :robot: Test - [x] :computer: CI/Deployment ## Test Plan - [x] :muscle: Manual --- .github/workflows/ci.yml | 3 ++- .github/workflows/load_test.yml | 1 + src/pkg/logger/logger.go | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 035c3ee4d..9739788cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,6 +208,7 @@ jobs: CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} + CORSO_LOG_FILE: stderr LOG_GRAPH_REQUESTS: true run: | set -euo pipefail @@ -219,7 +220,7 @@ jobs: -p 1 \ ./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests - # Upload the original go test log as an artifact for later review. + # Upload the original go test output as an artifact for later review. - name: Upload test log if: failure() uses: actions/upload-artifact@v3 diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index c5c8faec1..ced4a80af 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -53,6 +53,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} CORSO_LOAD_TESTS: true + CORSO_LOG_FILE: stderr CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }} CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }} CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }} diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index bead584b5..f64febe2e 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -106,6 +106,11 @@ func PreloadLoggingFlags() (string, string) { return "info", dlf } + // if not specified, attempt to fall back to env declaration. + if len(logfile) == 0 { + logfile = os.Getenv("CORSO_LOG_FILE") + } + if logfile == "-" { logfile = "stdout" } From 39c3c8a895c24525cc961a7bc8d339f919b78d1f Mon Sep 17 00:00:00 2001 From: Keepers Date: Mon, 23 Jan 2023 14:38:09 -0700 Subject: [PATCH 14/16] Issue 2022 eventincr (#2167) ## Description Fixes and unskips the tests incremental integration tests for exchange calendar events. Currently blocked on LynneR displaying the calendar fetch issues where it's only possible to retrieve her default Calendar, Birthdays, and US Holidays calendars. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :robot: Test ## Issue(s) * #2022 ## Test Plan - [x] :muscle: Manual - [x] :green_heart: E2E --- .../operations/backup_integration_test.go | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 3ee5c0230..2057eb561 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -643,8 +643,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { categories = map[path.CategoryType][]string{ path.EmailCategory: exchange.MetadataFileNames(path.EmailCategory), path.ContactsCategory: exchange.MetadataFileNames(path.ContactsCategory), - // TODO: not currently functioning; cannot retrieve generated calendars - // path.EventsCategory: exchange.MetadataFileNames(path.EventsCategory), + path.EventsCategory: exchange.MetadataFileNames(path.EventsCategory), } container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now) container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now) @@ -715,14 +714,13 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { container2: {}, }, }, - // TODO: not currently functioning; cannot retrieve generated calendars - // path.EventsCategory: { - // dbf: eventDBF, - // dests: map[string]contDeets{ - // container1: {}, - // container2: {}, - // }, - // }, + path.EventsCategory: { + dbf: eventDBF, + dests: map[string]contDeets{ + container1: {}, + container2: {}, + }, + }, } // populate initial test data @@ -775,6 +773,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { sel.Include( sel.MailFolders(containers, selectors.PrefixMatch()), sel.ContactFolders(containers, selectors.PrefixMatch()), + sel.EventCalendars(containers, selectors.PrefixMatch()), ) bo, _, kw, ms, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, ffs) @@ -883,8 +882,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { dataset[category].dests[container3] = contDeets{id, deets} } }, - itemsRead: 4, - itemsWritten: 4, + itemsRead: 6, // two items per category + itemsWritten: 6, }, { name: "rename a folder", @@ -934,8 +933,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { } } }, - itemsRead: 0, - itemsWritten: 4, + itemsRead: 2, + itemsWritten: 8, // two items per category }, { name: "add a new item", @@ -971,8 +970,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { } } }, - itemsRead: 2, - itemsWritten: 2, + itemsRead: 3, // one item per category + itemsWritten: 3, }, { name: "delete an existing item", @@ -1003,12 +1002,12 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { require.NoError(t, err, "getting event ids") require.NotEmpty(t, ids, "event ids in folder") - err = cli.CalendarsById(ids[0]).Delete(ctx, nil) + err = cli.EventsById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting calendar: %s", support.ConnectorStackErrorTrace(err)) } } }, - itemsRead: 2, + itemsRead: 3, // one item per category itemsWritten: 0, // deletes are not counted as "writes" }, } @@ -1035,9 +1034,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ) // do some additional checks to ensure the incremental dealt with fewer items. - // +4 on read/writes to account for metadata: 1 delta and 1 path for each type. - assert.Equal(t, test.itemsWritten+4, incBO.Results.ItemsWritten, "incremental items written") - assert.Equal(t, test.itemsRead+4, incBO.Results.ItemsRead, "incremental items read") + // +6 on read/writes to account for metadata: 1 delta and 1 path for each type. + assert.Equal(t, test.itemsWritten+6, incBO.Results.ItemsWritten, "incremental items written") + assert.Equal(t, test.itemsRead+6, incBO.Results.ItemsRead, "incremental items read") assert.NoError(t, incBO.Results.ReadErrors, "incremental read errors") assert.NoError(t, incBO.Results.WriteErrors, "incremental write errors") assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events") From ac73d874f9a57b0522189222f7d3b29c9353798d Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Mon, 23 Jan 2023 17:06:20 -0800 Subject: [PATCH 15/16] Update kopia/switch back to upstream repo (#2235) ## Description Update kopia version since upstream now includes recent PRs from our repo Upstream now includes the following: * io.ReadCloser interface for StreamingFiles * properly pass file size for cached StreamingFiles ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup ## Issue(s) * closes #2234 * closes #1732 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/go.mod | 6 ++---- src/go.sum | 8 ++++---- src/internal/kopia/upload_test.go | 24 ++++++++++++------------ 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/go.mod b/src/go.mod index b46328257..66ffdd7ef 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,15 +2,13 @@ module github.com/alcionai/corso/src go 1.19 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c - require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/aws/aws-sdk-go v1.44.184 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/kopia/kopia v0.12.2-0.20221229232524-ba938cf58cc8 + github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb github.com/microsoft/kiota-abstractions-go v0.15.2 github.com/microsoft/kiota-authentication-azure-go v0.5.0 github.com/microsoft/kiota-http-go v0.11.0 @@ -112,7 +110,7 @@ require ( go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/crypto v0.5.0 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/src/go.sum b/src/go.sum index ea0ca662c..6563d813b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -52,8 +52,6 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c h1:uUcdEZ4sz7kRYVWB3K49MBHdICRyXCVAzd4ZiY3lvo0= -github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c/go.mod h1:yzJV11S6N6XMboXt7oCO6Jy2jJHPeSMtA+KOJ9Y1548= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -241,6 +239,8 @@ github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747h github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kopia/htmluibuild v0.0.0-20220928042710-9fdd02afb1e7 h1:WP5VfIQL7AaYkO4zTNSCsVOawTzudbc4tvLojvg0RKc= +github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb h1:0jLaKLiloYvRNbuHHpnQkJ7STAgzQ4z6n+KPa6Kyg7I= +github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb/go.mod h1:dtCyMCsWulG82o9bDopvnny9DpOQe0PnSDczJLuhnWA= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -448,8 +448,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index ff86db13b..57ce9fd56 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -1027,7 +1027,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() { virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), }, ), @@ -1333,7 +1333,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(inboxFileName1)[0], time.Time{}, - bytes.NewReader(inboxFileData1), + io.NopCloser(bytes.NewReader(inboxFileData1)), ), virtualfs.NewStaticDirectory( encodeElements(personalDir)[0], @@ -1341,12 +1341,12 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(personalFileName1)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), virtualfs.StreamingFileWithModTimeFromReader( encodeElements(personalFileName2)[0], time.Time{}, - bytes.NewReader(testFileData2), + io.NopCloser(bytes.NewReader(testFileData2)), ), }, ), @@ -1356,7 +1356,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(workFileName1)[0], time.Time{}, - bytes.NewReader(testFileData3), + io.NopCloser(bytes.NewReader(testFileData3)), ), }, ), @@ -1973,7 +1973,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), }, ), @@ -1983,7 +1983,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName2)[0], time.Time{}, - bytes.NewReader(testFileData2), + io.NopCloser(bytes.NewReader(testFileData2)), ), }, ), @@ -1998,7 +1998,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName3)[0], time.Time{}, - bytes.NewReader(testFileData3), + io.NopCloser(bytes.NewReader(testFileData3)), ), }, ), @@ -2008,7 +2008,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName4)[0], time.Time{}, - bytes.NewReader(testFileData4), + io.NopCloser(bytes.NewReader(testFileData4)), ), }, ), @@ -2155,7 +2155,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(inboxFileName1)[0], time.Time{}, - bytes.NewReader(inboxFileData1), + io.NopCloser(bytes.NewReader(inboxFileData1)), ), }, ), @@ -2170,7 +2170,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(contactsFileName1)[0], time.Time{}, - bytes.NewReader(contactsFileData1), + io.NopCloser(bytes.NewReader(contactsFileData1)), ), }, ), @@ -2228,7 +2228,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(eventsFileName1)[0], time.Time{}, - bytes.NewReader(eventsFileData1), + io.NopCloser(bytes.NewReader(eventsFileData1)), ), }, ), From 4a6959f60ff3976cb4ef3f7852a7a7591d4febf8 Mon Sep 17 00:00:00 2001 From: Vaibhav Kamra Date: Wed, 25 Jan 2023 02:21:00 -0800 Subject: [PATCH 16/16] Revert "Issue 2022 eventincr" (#2256) Reverts alcionai/corso#2167. This is causing the integration test to fail consistently (in CI and local runs) --- .../operations/backup_integration_test.go | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 2057eb561..3ee5c0230 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -643,7 +643,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { categories = map[path.CategoryType][]string{ path.EmailCategory: exchange.MetadataFileNames(path.EmailCategory), path.ContactsCategory: exchange.MetadataFileNames(path.ContactsCategory), - path.EventsCategory: exchange.MetadataFileNames(path.EventsCategory), + // TODO: not currently functioning; cannot retrieve generated calendars + // path.EventsCategory: exchange.MetadataFileNames(path.EventsCategory), } container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now) container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now) @@ -714,13 +715,14 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { container2: {}, }, }, - path.EventsCategory: { - dbf: eventDBF, - dests: map[string]contDeets{ - container1: {}, - container2: {}, - }, - }, + // TODO: not currently functioning; cannot retrieve generated calendars + // path.EventsCategory: { + // dbf: eventDBF, + // dests: map[string]contDeets{ + // container1: {}, + // container2: {}, + // }, + // }, } // populate initial test data @@ -773,7 +775,6 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { sel.Include( sel.MailFolders(containers, selectors.PrefixMatch()), sel.ContactFolders(containers, selectors.PrefixMatch()), - sel.EventCalendars(containers, selectors.PrefixMatch()), ) bo, _, kw, ms, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, ffs) @@ -882,8 +883,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { dataset[category].dests[container3] = contDeets{id, deets} } }, - itemsRead: 6, // two items per category - itemsWritten: 6, + itemsRead: 4, + itemsWritten: 4, }, { name: "rename a folder", @@ -933,8 +934,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { } } }, - itemsRead: 2, - itemsWritten: 8, // two items per category + itemsRead: 0, + itemsWritten: 4, }, { name: "add a new item", @@ -970,8 +971,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { } } }, - itemsRead: 3, // one item per category - itemsWritten: 3, + itemsRead: 2, + itemsWritten: 2, }, { name: "delete an existing item", @@ -1002,12 +1003,12 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { require.NoError(t, err, "getting event ids") require.NotEmpty(t, ids, "event ids in folder") - err = cli.EventsById(ids[0]).Delete(ctx, nil) + err = cli.CalendarsById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting calendar: %s", support.ConnectorStackErrorTrace(err)) } } }, - itemsRead: 3, // one item per category + itemsRead: 2, itemsWritten: 0, // deletes are not counted as "writes" }, } @@ -1034,9 +1035,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ) // do some additional checks to ensure the incremental dealt with fewer items. - // +6 on read/writes to account for metadata: 1 delta and 1 path for each type. - assert.Equal(t, test.itemsWritten+6, incBO.Results.ItemsWritten, "incremental items written") - assert.Equal(t, test.itemsRead+6, incBO.Results.ItemsRead, "incremental items read") + // +4 on read/writes to account for metadata: 1 delta and 1 path for each type. + assert.Equal(t, test.itemsWritten+4, incBO.Results.ItemsWritten, "incremental items written") + assert.Equal(t, test.itemsRead+4, incBO.Results.ItemsRead, "incremental items read") assert.NoError(t, incBO.Results.ReadErrors, "incremental read errors") assert.NoError(t, incBO.Results.WriteErrors, "incremental write errors") assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events")