diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 035c3ee4d..9739788cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,6 +208,7 @@ jobs: CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} + CORSO_LOG_FILE: stderr LOG_GRAPH_REQUESTS: true run: | set -euo pipefail @@ -219,7 +220,7 @@ jobs: -p 1 \ ./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests - # Upload the original go test log as an artifact for later review. + # Upload the original go test output as an artifact for later review. - name: Upload test log if: failure() uses: actions/upload-artifact@v3 diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index c5c8faec1..ced4a80af 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -53,6 +53,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} CORSO_LOAD_TESTS: true + CORSO_LOG_FILE: stderr CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }} CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }} CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }} diff --git a/README.md b/README.md index cdb00ddf3..33341e4e7 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ services, possibly beyond M365, will expand based on the interest and needs of t # Getting Started -See the [Corso Documentation](https://corsobackup.io/docs/intro) for more information. +See the [Corso Quickstart](https://corsobackup.io/docs/quickstart/) on our docs page. # Building Corso diff --git a/src/go.mod b/src/go.mod index 1f3017de0..9c031a21c 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,15 +2,13 @@ module github.com/alcionai/corso/src go 1.19 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c - require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.183 + github.com/aws/aws-sdk-go v1.44.186 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/kopia/kopia v0.12.2-0.20221229232524-ba938cf58cc8 + github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb github.com/microsoft/kiota-abstractions-go v0.15.2 github.com/microsoft/kiota-authentication-azure-go v0.5.0 github.com/microsoft/kiota-http-go v0.11.0 @@ -112,7 +110,7 @@ require ( go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/crypto v0.5.0 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/src/go.sum b/src/go.sum index 8650e7297..f1407e45f 100644 --- a/src/go.sum +++ b/src/go.sum @@ -52,8 +52,6 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c h1:uUcdEZ4sz7kRYVWB3K49MBHdICRyXCVAzd4ZiY3lvo0= -github.com/alcionai/kopia v0.10.8-0.20230112200734-ac706ef83a1c/go.mod h1:yzJV11S6N6XMboXt7oCO6Jy2jJHPeSMtA+KOJ9Y1548= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -62,8 +60,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk= -github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.186 h1:HInpD2b9FXgJIcP/WDRuSW4Wri9i5WVglO9okFFuOow= +github.com/aws/aws-sdk-go v1.44.186/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -241,6 +239,8 @@ github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747h github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kopia/htmluibuild v0.0.0-20220928042710-9fdd02afb1e7 h1:WP5VfIQL7AaYkO4zTNSCsVOawTzudbc4tvLojvg0RKc= +github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb h1:0jLaKLiloYvRNbuHHpnQkJ7STAgzQ4z6n+KPa6Kyg7I= +github.com/kopia/kopia v0.12.2-0.20230123092305-e5387cec0acb/go.mod h1:dtCyMCsWulG82o9bDopvnny9DpOQe0PnSDczJLuhnWA= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -448,8 +448,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index e168ed082..ce37beab6 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -257,6 +257,24 @@ func (col *Collection) streamItems(ctx context.Context) { break } + // If the data is no longer available just return here and chalk it up + // as a success. There's no reason to retry and no way we can backup up + // enough information to restore the item anyway. + if e := graph.IsErrDeletedInFlight(err); e != nil { + atomic.AddInt64(&success, 1) + logger.Ctx(ctx).Infow( + "Graph reported item not found", + "error", + e, + "service", + path.ExchangeService.String(), + "category", + col.category.String, + ) + + return + } + if i < numberOfRetries { time.Sleep(time.Duration(3*(i+1)) * time.Second) } @@ -270,6 +288,16 @@ func (col *Collection) streamItems(ctx context.Context) { // attempted items. if e := graph.IsErrDeletedInFlight(err); e != nil { atomic.AddInt64(&success, 1) + logger.Ctx(ctx).Infow( + "Graph reported item not found", + "error", + e, + "service", + path.ExchangeService.String(), + "category", + col.category.String, + ) + return } diff --git a/src/internal/connector/support/status.go b/src/internal/connector/support/status.go index fb08ea1c4..3f2435263 100644 --- a/src/internal/connector/support/status.go +++ b/src/internal/connector/support/status.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + multierror "github.com/hashicorp/go-multierror" bytesize "github.com/inhies/go-bytesize" "github.com/alcionai/corso/src/pkg/logger" @@ -21,6 +22,7 @@ type ConnectorOperationStatus struct { FolderCount int Successful int ErrorCount int + Err error incomplete bool incompleteReason string additionalDetails string @@ -70,6 +72,7 @@ func CreateStatus( FolderCount: folders, Successful: cm.Successes, ErrorCount: numErr, + Err: err, incomplete: hasErrors, incompleteReason: reason, bytes: cm.TotalBytes, @@ -115,6 +118,7 @@ func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus { FolderCount: one.FolderCount + two.FolderCount, Successful: one.Successful + two.Successful, ErrorCount: one.ErrorCount + two.ErrorCount, + Err: multierror.Append(one.Err, two.Err).ErrorOrNil(), bytes: one.bytes + two.bytes, incomplete: hasErrors, incompleteReason: one.incompleteReason + ", " + two.incompleteReason, diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index ff86db13b..57ce9fd56 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -1027,7 +1027,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() { virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), }, ), @@ -1333,7 +1333,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(inboxFileName1)[0], time.Time{}, - bytes.NewReader(inboxFileData1), + io.NopCloser(bytes.NewReader(inboxFileData1)), ), virtualfs.NewStaticDirectory( encodeElements(personalDir)[0], @@ -1341,12 +1341,12 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(personalFileName1)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), virtualfs.StreamingFileWithModTimeFromReader( encodeElements(personalFileName2)[0], time.Time{}, - bytes.NewReader(testFileData2), + io.NopCloser(bytes.NewReader(testFileData2)), ), }, ), @@ -1356,7 +1356,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto virtualfs.StreamingFileWithModTimeFromReader( encodeElements(workFileName1)[0], time.Time{}, - bytes.NewReader(testFileData3), + io.NopCloser(bytes.NewReader(testFileData3)), ), }, ), @@ -1973,7 +1973,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName)[0], time.Time{}, - bytes.NewReader(testFileData), + io.NopCloser(bytes.NewReader(testFileData)), ), }, ), @@ -1983,7 +1983,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName2)[0], time.Time{}, - bytes.NewReader(testFileData2), + io.NopCloser(bytes.NewReader(testFileData2)), ), }, ), @@ -1998,7 +1998,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName3)[0], time.Time{}, - bytes.NewReader(testFileData3), + io.NopCloser(bytes.NewReader(testFileData3)), ), }, ), @@ -2008,7 +2008,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre virtualfs.StreamingFileWithModTimeFromReader( encodeElements(testFileName4)[0], time.Time{}, - bytes.NewReader(testFileData4), + io.NopCloser(bytes.NewReader(testFileData4)), ), }, ), @@ -2155,7 +2155,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(inboxFileName1)[0], time.Time{}, - bytes.NewReader(inboxFileData1), + io.NopCloser(bytes.NewReader(inboxFileData1)), ), }, ), @@ -2170,7 +2170,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(contactsFileName1)[0], time.Time{}, - bytes.NewReader(contactsFileData1), + io.NopCloser(bytes.NewReader(contactsFileData1)), ), }, ), @@ -2228,7 +2228,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt virtualfs.StreamingFileWithModTimeFromReader( encodeElements(eventsFileName1)[0], time.Time{}, - bytes.NewReader(eventsFileData1), + io.NopCloser(bytes.NewReader(eventsFileData1)), ), }, ), diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 92a8b93c6..e2c2bd0eb 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -208,13 +208,11 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { opStats.gc = gc.AwaitStatus() if opStats.gc.ErrorCount > 0 { - opStats.writeErr = multierror.Append(nil, opStats.writeErr, errors.Errorf( - "%v errors reported while fetching item data", - opStats.gc.ErrorCount, - )).ErrorOrNil() + merr := multierror.Append(opStats.readErr, errors.Wrap(opStats.gc.Err, "retrieving data")) + opStats.readErr = merr.ErrorOrNil() // Need to exit before we set started to true else we'll report no errors. - return opStats.writeErr + return opStats.readErr } // should always be 1, since backups are 1:1 with resourceOwners. diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index bead584b5..f64febe2e 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -106,6 +106,11 @@ func PreloadLoggingFlags() (string, string) { return "info", dlf } + // if not specified, attempt to fall back to env declaration. + if len(logfile) == 0 { + logfile = os.Getenv("CORSO_LOG_FILE") + } + if logfile == "-" { logfile = "stdout" } diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 34aabb2c5..4ce3859cd 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -193,12 +193,13 @@ func (suite *SharePointSelectorSuite) TestToSharePointRestore() { func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { var ( - pairA = "folderA/folderC" - item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderA/folderB", "item") - item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", pairA, "item2") - item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderD/folderE", "item3") - item4 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", "folderG/folderH", "item4") - item5 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", "folderG/folderH", "item5") + pairAC = "folderA/folderC" + pairGH = "folderG/folderH" + item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderA/folderB", "item") + item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", pairAC, "item2") + item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "sid", "folderD/folderE", "item3") + item4 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item4") + item5 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item5") ) deets := &details.Details{ @@ -283,7 +284,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets: deets, makeSelector: func() *SharePointRestore { odr := NewSharePointRestore([]string{"sid"}) - odr.Include(odr.Libraries([]string{"folderA/folderB", pairA})) + odr.Include(odr.Libraries([]string{"folderA/folderB", pairAC})) return odr }, expect: arr(item, item2), @@ -293,7 +294,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets: deets, makeSelector: func() *SharePointRestore { odr := NewSharePointRestore([]string{"sid"}) - odr.Include(odr.Pages([]string{"folderG/folderH", pairA})) + odr.Include(odr.Pages([]string{pairGH, pairAC})) return odr }, expect: arr(item4, item5), diff --git a/website/package-lock.json b/website/package-lock.json index c781b7458..0662eb756 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -13220,9 +13220,9 @@ } }, "node_modules/ua-parser-js": { - "version": "0.7.32", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.32.tgz", - "integrity": "sha512-f9BESNVhzlhEFf2CHMSj40NWOjYPl1YKYbrvIr/hFTDEmLq7SRbWvm7FcdcpCYT95zrOhC7gZSxjdnnTpBcwVw==", + "version": "0.7.33", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.33.tgz", + "integrity": "sha512-s8ax/CeZdK9R/56Sui0WM6y9OFREJarMRHqLB2EwkovemBxNQ+Bqu8GAsUnVcXKgphb++ghr/B2BZx4mahujPw==", "funding": [ { "type": "opencollective", @@ -13233,7 +13233,6 @@ "url": "https://paypal.me/faisalman" } ], - "license": "MIT", "engines": { "node": "*" } @@ -23418,9 +23417,9 @@ "peer": true }, "ua-parser-js": { - "version": "0.7.32", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.32.tgz", - "integrity": "sha512-f9BESNVhzlhEFf2CHMSj40NWOjYPl1YKYbrvIr/hFTDEmLq7SRbWvm7FcdcpCYT95zrOhC7gZSxjdnnTpBcwVw==" + "version": "0.7.33", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.33.tgz", + "integrity": "sha512-s8ax/CeZdK9R/56Sui0WM6y9OFREJarMRHqLB2EwkovemBxNQ+Bqu8GAsUnVcXKgphb++ghr/B2BZx4mahujPw==" }, "unherit": { "version": "1.1.3",