larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into sharepoint-restore-selectors

Browse Source
This commit is contained in:
Danny Adams 2023-01-25 07:50:47 -05:00
commit c7828db7d6
12 changed files with 80 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/ci.yml vendored
View File

@ -208,6 +208,7 @@ jobs:
CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
CORSO_LOG_FILE: stderr
LOG_GRAPH_REQUESTS: true
run: |
set -euo pipefail
@ -219,7 +220,7 @@ jobs:
-p 1 \
./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests
# Upload the original go test log as an artifact for later review.
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3

1
.github/workflows/load_test.yml vendored
View File

@ -53,6 +53,7 @@ jobs:
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
CORSO_LOAD_TESTS: true
CORSO_LOG_FILE: stderr
CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }}

2
README.md
View File

@ -20,7 +20,7 @@ services, possibly beyond M365, will expand based on the interest and needs of t
# Getting Started
See the [Corso Documentation](https://corsobackup.io/docs/intro) for more information.
See the [Corso Quickstart](https://corsobackup.io/docs/quickstart/) on our docs page.
# Building Corso

8
src/go.mod
View File

@ -2,15 +2,13 @@ module github.com/alcionai/corso/src
go 1.19
replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c
require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
github.com/aws/aws-sdk-go v1.44.183
github.com/aws/aws-sdk-go v1.44.186
github.com/aws/aws-xray-sdk-go v1.8.0
github.com/google/uuid v1.3.0
github.com/hashicorp/go-multierror v1.1.1
github.com/kopia/kopia v0.12.2-0.20221229232524-ba938cf58cc8
github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb
github.com/microsoft/kiota-abstractions-go v0.15.2
github.com/microsoft/kiota-authentication-azure-go v0.5.0
github.com/microsoft/kiota-http-go v0.11.0
@ -112,7 +110,7 @@ require (
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sync v0.1.0 // indirect

12
src/go.sum
View File

@ -52,8 +52,6 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c h1:uUcdEZ4sz7kRYVWB3K49MBHdICRyXCVAzd4ZiY3lvo0=
github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c/go.mod h1:yzJV11S6N6XMboXt7oCO6Jy2jJHPeSMtA+KOJ9Y1548=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@ -62,8 +60,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk=
github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.186 h1:HInpD2b9FXgJIcP/WDRuSW4Wri9i5WVglO9okFFuOow=
github.com/aws/aws-sdk-go v1.44.186/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY=
github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@ -241,6 +239,8 @@ github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747h
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kopia/htmluibuild v0.0.0-20220928042710-9fdd02afb1e7 h1:WP5VfIQL7AaYkO4zTNSCsVOawTzudbc4tvLojvg0RKc=
github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb h1:0jLaKLiloYvRNbuHHpnQkJ7STAgzQ4z6n+KPa6Kyg7I=
github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb/go.mod h1:dtCyMCsWulG82o9bDopvnny9DpOQe0PnSDczJLuhnWA=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -448,8 +448,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

28
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -257,6 +257,24 @@ func (col *Collection) streamItems(ctx context.Context) {
break
}
// If the data is no longer available just return here and chalk it up
// as a success. There's no reason to retry and no way we can backup up
// enough information to restore the item anyway.
if e := graph.IsErrDeletedInFlight(err); e != nil {
atomic.AddInt64(&success, 1)
logger.Ctx(ctx).Infow(
"Graph reported item not found",
"error",
e,
"service",
path.ExchangeService.String(),
"category",
col.category.String,
)
return
}
if i < numberOfRetries {
time.Sleep(time.Duration(3*(i+1)) * time.Second)
}
@ -270,6 +288,16 @@ func (col *Collection) streamItems(ctx context.Context) {
// attempted items.
if e := graph.IsErrDeletedInFlight(err); e != nil {
atomic.AddInt64(&success, 1)
logger.Ctx(ctx).Infow(
"Graph reported item not found",
"error",
e,
"service",
path.ExchangeService.String(),
"category",
col.category.String,
)
return
}

4
src/internal/connector/support/status.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
multierror "github.com/hashicorp/go-multierror"
bytesize "github.com/inhies/go-bytesize"
"github.com/alcionai/corso/src/pkg/logger"
@ -21,6 +22,7 @@ type ConnectorOperationStatus struct {
FolderCount int
Successful int
ErrorCount int
Err error
incomplete bool
incompleteReason string
additionalDetails string
@ -70,6 +72,7 @@ func CreateStatus(
FolderCount: folders,
Successful: cm.Successes,
ErrorCount: numErr,
Err: err,
incomplete: hasErrors,
incompleteReason: reason,
bytes: cm.TotalBytes,
@ -115,6 +118,7 @@ func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus {
FolderCount: one.FolderCount + two.FolderCount,
Successful: one.Successful + two.Successful,
ErrorCount: one.ErrorCount + two.ErrorCount,
Err: multierror.Append(one.Err, two.Err).ErrorOrNil(),
bytes: one.bytes + two.bytes,
incomplete: hasErrors,
incompleteReason: one.incompleteReason + ", " + two.incompleteReason,

24
src/internal/kopia/upload_test.go
View File

@ -1027,7 +1027,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() {
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName)[0],
time.Time{},
bytes.NewReader(testFileData),
io.NopCloser(bytes.NewReader(testFileData)),
),
},
),
@ -1333,7 +1333,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(inboxFileName1)[0],
time.Time{},
bytes.NewReader(inboxFileData1),
io.NopCloser(bytes.NewReader(inboxFileData1)),
),
virtualfs.NewStaticDirectory(
encodeElements(personalDir)[0],
@ -1341,12 +1341,12 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(personalFileName1)[0],
time.Time{},
bytes.NewReader(testFileData),
io.NopCloser(bytes.NewReader(testFileData)),
),
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(personalFileName2)[0],
time.Time{},
bytes.NewReader(testFileData2),
io.NopCloser(bytes.NewReader(testFileData2)),
),
},
),
@ -1356,7 +1356,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(workFileName1)[0],
time.Time{},
bytes.NewReader(testFileData3),
io.NopCloser(bytes.NewReader(testFileData3)),
),
},
),
@ -1973,7 +1973,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName)[0],
time.Time{},
bytes.NewReader(testFileData),
io.NopCloser(bytes.NewReader(testFileData)),
),
},
),
@ -1983,7 +1983,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName2)[0],
time.Time{},
bytes.NewReader(testFileData2),
io.NopCloser(bytes.NewReader(testFileData2)),
),
},
),
@ -1998,7 +1998,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName3)[0],
time.Time{},
bytes.NewReader(testFileData3),
io.NopCloser(bytes.NewReader(testFileData3)),
),
},
),
@ -2008,7 +2008,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(testFileName4)[0],
time.Time{},
bytes.NewReader(testFileData4),
io.NopCloser(bytes.NewReader(testFileData4)),
),
},
),
@ -2155,7 +2155,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(inboxFileName1)[0],
time.Time{},
bytes.NewReader(inboxFileData1),
io.NopCloser(bytes.NewReader(inboxFileData1)),
),
},
),
@ -2170,7 +2170,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(contactsFileName1)[0],
time.Time{},
bytes.NewReader(contactsFileData1),
io.NopCloser(bytes.NewReader(contactsFileData1)),
),
},
),
@ -2228,7 +2228,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
virtualfs.StreamingFileWithModTimeFromReader(
encodeElements(eventsFileName1)[0],
time.Time{},
bytes.NewReader(eventsFileData1),
io.NopCloser(bytes.NewReader(eventsFileData1)),
),
},
),

8
src/internal/operations/backup.go
View File

@ -208,13 +208,11 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
opStats.gc = gc.AwaitStatus()
if opStats.gc.ErrorCount > 0 {
opStats.writeErr = multierror.Append(nil, opStats.writeErr, errors.Errorf(
"%v errors reported while fetching item data",
opStats.gc.ErrorCount,
)).ErrorOrNil()
merr := multierror.Append(opStats.readErr, errors.Wrap(opStats.gc.Err, "retrieving data"))
opStats.readErr = merr.ErrorOrNil()
// Need to exit before we set started to true else we'll report no errors.
return opStats.writeErr
return opStats.readErr
}
// should always be 1, since backups are 1:1 with resourceOwners.

5
src/pkg/logger/logger.go
View File

@ -106,6 +106,11 @@ func PreloadLoggingFlags() (string, string) {
return "info", dlf
}
// if not specified, attempt to fall back to env declaration.
if len(logfile) == 0 {
logfile = os.Getenv("CORSO_LOG_FILE")
}
if logfile == "-" {
logfile = "stdout"
}

17
src/pkg/selectors/sharepoint_test.go
View File

@ -193,12 +193,13 @@ func (suite *SharePointSelectorSuite) TestToSharePointRestore() {
func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
var (
pairA = "folderA/folderC"
item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderA/folderB", "item")
item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", pairA, "item2")
item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderD/folderE", "item3")
item4 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", "folderG/folderH", "item4")
item5 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", "folderG/folderH", "item5")
pairAC = "folderA/folderC"
pairGH = "folderG/folderH"
item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderA/folderB", "item")
item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", pairAC, "item2")
item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderD/folderE", "item3")
item4 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item4")
item5 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item5")
)
deets := &details.Details{
@ -283,7 +284,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore([]string{"sid"})
odr.Include(odr.Libraries([]string{"folderA/folderB", pairA}))
odr.Include(odr.Libraries([]string{"folderA/folderB", pairAC}))
return odr
},
expect: arr(item, item2),
@ -293,7 +294,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
deets: deets,
makeSelector: func() *SharePointRestore {
odr := NewSharePointRestore([]string{"sid"})
odr.Include(odr.Pages([]string{"folderG/folderH", pairA}))
odr.Include(odr.Pages([]string{pairGH, pairAC}))
return odr
},
expect: arr(item4, item5),

13
website/package-lock.json generated
View File

@ -13220,9 +13220,9 @@
}
},
"node_modules/ua-parser-js": {
"version": "0.7.32",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.32.tgz",
"integrity": "sha512-f9BESNVhzlhEFf2CHMSj40NWOjYPl1YKYbrvIr/hFTDEmLq7SRbWvm7FcdcpCYT95zrOhC7gZSxjdnnTpBcwVw==",
"version": "0.7.33",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.33.tgz",
"integrity": "sha512-s8ax/CeZdK9R/56Sui0WM6y9OFREJarMRHqLB2EwkovemBxNQ+Bqu8GAsUnVcXKgphb++ghr/B2BZx4mahujPw==",
"funding": [
{
"type": "opencollective",
@ -13233,7 +13233,6 @@
"url": "https://paypal.me/faisalman"
}
],
"license": "MIT",
"engines": {
"node": "*"
}
@ -23418,9 +23417,9 @@
"peer": true
},
"ua-parser-js": {
"version": "0.7.32",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.32.tgz",
"integrity": "sha512-f9BESNVhzlhEFf2CHMSj40NWOjYPl1YKYbrvIr/hFTDEmLq7SRbWvm7FcdcpCYT95zrOhC7gZSxjdnnTpBcwVw=="
"version": "0.7.33",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.33.tgz",
"integrity": "sha512-s8ax/CeZdK9R/56Sui0WM6y9OFREJarMRHqLB2EwkovemBxNQ+Bqu8GAsUnVcXKgphb++ghr/B2BZx4mahujPw=="
},
"unherit": {
"version": "1.1.3",