From 06b04c6eff352703ed541eaf6347cd94270b5da8 Mon Sep 17 00:00:00 2001 From: neha_gupta Date: Wed, 31 May 2023 23:32:38 +0530 Subject: [PATCH 01/10] update files count and total size on CLI for backup (#3480) Currently the backup output show, - total backup size - size of files + size of metafiles - no of files - no of files + no of metafile With this change the output of backup command shows size of only files and count include only file count. NOTE: all current three services results will be impacted here. #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :bug: Bugfix #### Issue(s) * https://github.com/alcionai/corso/issues/3304 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- src/internal/kopia/wrapper.go | 16 ++-- src/internal/operations/backup.go | 13 +++ .../operations/backup_integration_test.go | 91 ++++++++++++------- src/internal/operations/backup_test.go | 12 ++- src/internal/stats/stats.go | 12 ++- src/pkg/backup/backup.go | 4 +- src/pkg/backup/backup_test.go | 12 ++- src/pkg/backup/details/details.go | 15 +++ 8 files changed, 120 insertions(+), 55 deletions(-) diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index e4d73bb4c..e92d2e133 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -50,14 +50,16 @@ var ( type BackupStats struct { SnapshotID string - TotalHashedBytes int64 - TotalUploadedBytes int64 + TotalHashedBytes int64 + TotalUploadedBytes int64 + TotalNonMetaUploadedBytes int64 - TotalFileCount int - CachedFileCount int - UncachedFileCount int - TotalDirectoryCount int - ErrorCount int + TotalFileCount int + TotalNonMetaFileCount int + CachedFileCount int + UncachedFileCount int + TotalDirectoryCount int + ErrorCount int IgnoredErrorCount int ExpectedIgnoredErrorCount int diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 8e4a8f25f..92e95c53b 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -347,6 +347,8 @@ func (op *BackupOperation) do( mans, toMerge, deets, + writeStats, + op.Selectors.PathService(), op.Errors) if err != nil { return nil, clues.Wrap(err, "merging details") @@ -704,8 +706,17 @@ func mergeDetails( mans []*kopia.ManifestEntry, dataFromBackup kopia.DetailsMergeInfoer, deets *details.Builder, + writeStats *kopia.BackupStats, + serviceType path.ServiceType, errs *fault.Bus, ) error { + detailsModel := deets.Details().DetailsModel + + // getting the values in writeStats before anything else so that we don't get a return from + // conditions like no backup data. + writeStats.TotalNonMetaFileCount = len(detailsModel.FilterMetaFiles().Items()) + writeStats.TotalNonMetaUploadedBytes = detailsModel.SumNonMetaFileSizes() + // Don't bother loading any of the base details if there's nothing we need to merge. if dataFromBackup == nil || dataFromBackup.ItemsToMerge() == 0 { return nil @@ -841,6 +852,8 @@ func (op *BackupOperation) persistResults( op.Results.BytesRead = opStats.k.TotalHashedBytes op.Results.BytesUploaded = opStats.k.TotalUploadedBytes op.Results.ItemsWritten = opStats.k.TotalFileCount + op.Results.NonMetaBytesUploaded = opStats.k.TotalNonMetaUploadedBytes + op.Results.NonMetaItemsWritten = opStats.k.TotalNonMetaFileCount op.Results.ResourceOwners = opStats.resourceCount if opStats.gc == nil { diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index df32d3fb5..e0db072c8 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -961,6 +961,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten int nonDeltaItemsRead int nonDeltaItemsWritten int + nonMetaItemsWritten int }{ { name: "clean, no changes", @@ -969,6 +970,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 0, nonDeltaItemsRead: 8, nonDeltaItemsWritten: 0, // unchanged items are not counted towards write + nonMetaItemsWritten: 4, }, { name: "move an email folder to a subfolder", @@ -992,6 +994,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 2, nonDeltaItemsRead: 8, nonDeltaItemsWritten: 2, + nonMetaItemsWritten: 6, }, { name: "delete a folder", @@ -1018,6 +1021,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 0, // deletions are not counted as "writes" nonDeltaItemsRead: 4, nonDeltaItemsWritten: 0, + nonMetaItemsWritten: 4, }, { name: "add a new folder", @@ -1070,6 +1074,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 4, nonDeltaItemsRead: 8, nonDeltaItemsWritten: 4, + nonMetaItemsWritten: 8, }, { name: "rename a folder", @@ -1125,6 +1130,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 0, // two items per category nonDeltaItemsRead: 8, nonDeltaItemsWritten: 0, + nonMetaItemsWritten: 4, }, { name: "add a new item", @@ -1178,6 +1184,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 2, nonDeltaItemsRead: 10, nonDeltaItemsWritten: 2, + nonMetaItemsWritten: 6, }, { name: "delete an existing item", @@ -1231,6 +1238,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont deltaItemsWritten: 0, // deletes are not counted as "writes" nonDeltaItemsRead: 8, nonDeltaItemsWritten: 0, + nonMetaItemsWritten: 4, }, } @@ -1263,7 +1271,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont assert.Equal(t, test.nonDeltaItemsRead+4, incBO.Results.ItemsRead, "non delta items read") assert.Equal(t, test.nonDeltaItemsWritten+4, incBO.Results.ItemsWritten, "non delta items written") } - + assert.Equal(t, test.nonMetaItemsWritten, incBO.Results.ItemsWritten, "non meta incremental items write") assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(incBO.Errors.Failure())) assert.Empty(t, incBO.Errors.Recovered(), "incremental recoverable/iteration errors") assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events") @@ -1527,9 +1535,10 @@ func runDriveIncrementalTest( table := []struct { name string // performs the incremental update required for the test. - updateFiles func(t *testing.T) - itemsRead int - itemsWritten int + updateFiles func(t *testing.T) + itemsRead int + itemsWritten int + nonMetaItemsWritten int }{ { name: "clean incremental, no changes", @@ -1556,8 +1565,9 @@ func runDriveIncrementalTest( expectDeets.AddItem(driveID, makeLocRef(container1), newFileID) }, - itemsRead: 1, // .data file for newitem - itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + itemsRead: 1, // .data file for newitem + itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + nonMetaItemsWritten: 1, // .data file for newitem }, { name: "add permission to new file", @@ -1578,8 +1588,9 @@ func runDriveIncrementalTest( require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) // no expectedDeets: metadata isn't tracked }, - itemsRead: 1, // .data file for newitem - itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) + itemsRead: 1, // .data file for newitem + itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) + nonMetaItemsWritten: 1, // the file for which permission was updated }, { name: "remove permission from new file", @@ -1599,8 +1610,9 @@ func runDriveIncrementalTest( require.NoErrorf(t, err, "removing permission from file %v", clues.ToCore(err)) // no expectedDeets: metadata isn't tracked }, - itemsRead: 1, // .data file for newitem - itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) + itemsRead: 1, // .data file for newitem + itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) + nonMetaItemsWritten: 1, //.data file for newitem }, { name: "add permission to container", @@ -1621,8 +1633,9 @@ func runDriveIncrementalTest( require.NoErrorf(t, err, "adding permission to container %v", clues.ToCore(err)) // no expectedDeets: metadata isn't tracked }, - itemsRead: 0, - itemsWritten: 1, // .dirmeta for collection + itemsRead: 0, + itemsWritten: 1, // .dirmeta for collection + nonMetaItemsWritten: 0, // no files updated as update on container }, { name: "remove permission from container", @@ -1643,8 +1656,9 @@ func runDriveIncrementalTest( require.NoErrorf(t, err, "removing permission from container %v", clues.ToCore(err)) // no expectedDeets: metadata isn't tracked }, - itemsRead: 0, - itemsWritten: 1, // .dirmeta for collection + itemsRead: 0, + itemsWritten: 1, // .dirmeta for collection + nonMetaItemsWritten: 0, // no files updated }, { name: "update contents of a file", @@ -1658,8 +1672,9 @@ func runDriveIncrementalTest( require.NoErrorf(t, err, "updating file contents: %v", clues.ToCore(err)) // no expectedDeets: neither file id nor location changed }, - itemsRead: 1, // .data file for newitem - itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + itemsRead: 1, // .data file for newitem + itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + nonMetaItemsWritten: 1, // .data file for newitem }, { name: "rename a file", @@ -1681,8 +1696,9 @@ func runDriveIncrementalTest( driveItem) require.NoError(t, err, "renaming file %v", clues.ToCore(err)) }, - itemsRead: 1, // .data file for newitem - itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + itemsRead: 1, // .data file for newitem + itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + nonMetaItemsWritten: 1, // .data file for newitem // no expectedDeets: neither file id nor location changed }, { @@ -1710,8 +1726,9 @@ func runDriveIncrementalTest( makeLocRef(container2), ptr.Val(newFile.GetId())) }, - itemsRead: 1, // .data file for newitem - itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + itemsRead: 1, // .data file for newitem + itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + nonMetaItemsWritten: 1, // .data file for new item }, { name: "delete file", @@ -1725,8 +1742,9 @@ func runDriveIncrementalTest( expectDeets.RemoveItem(driveID, makeLocRef(container2), ptr.Val(newFile.GetId())) }, - itemsRead: 0, - itemsWritten: 0, + itemsRead: 0, + itemsWritten: 0, + nonMetaItemsWritten: 0, }, { name: "move a folder to a subfolder", @@ -1753,8 +1771,9 @@ func runDriveIncrementalTest( makeLocRef(container2), makeLocRef(container1)) }, - itemsRead: 0, - itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) + itemsRead: 0, + itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) + nonMetaItemsWritten: 0, }, { name: "rename a folder", @@ -1783,8 +1802,9 @@ func runDriveIncrementalTest( makeLocRef(container1, container2), makeLocRef(container1, containerRename)) }, - itemsRead: 0, - itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) + itemsRead: 0, + itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) + nonMetaItemsWritten: 0, }, { name: "delete a folder", @@ -1799,8 +1819,9 @@ func runDriveIncrementalTest( expectDeets.RemoveLocation(driveID, makeLocRef(container1, containerRename)) }, - itemsRead: 0, - itemsWritten: 0, + itemsRead: 0, + itemsWritten: 0, + nonMetaItemsWritten: 0, }, { name: "add a new folder", @@ -1831,8 +1852,9 @@ func runDriveIncrementalTest( expectDeets.AddLocation(driveID, container3) }, - itemsRead: 2, // 2 .data for 2 files - itemsWritten: 6, // read items + 2 directory meta + itemsRead: 2, // 2 .data for 2 files + itemsWritten: 6, // read items + 2 directory meta + nonMetaItemsWritten: 2, // 2 .data for 2 files }, } for _, test := range table { @@ -1862,9 +1884,10 @@ func runDriveIncrementalTest( // do some additional checks to ensure the incremental dealt with fewer items. // +2 on read/writes to account for metadata: 1 delta and 1 path. var ( - expectWrites = test.itemsWritten + 2 - expectReads = test.itemsRead + 2 - assertReadWrite = assert.Equal + expectWrites = test.itemsWritten + 2 + expectNonMetaWrites = test.nonMetaItemsWritten + expectReads = test.itemsRead + 2 + assertReadWrite = assert.Equal ) // Sharepoint can produce a superset of permissions by nature of @@ -1876,6 +1899,7 @@ func runDriveIncrementalTest( } assertReadWrite(t, expectWrites, incBO.Results.ItemsWritten, "incremental items written") + assertReadWrite(t, expectNonMetaWrites, incBO.Results.NonMetaItemsWritten, "incremental non-meta items written") assertReadWrite(t, expectReads, incBO.Results.ItemsRead, "incremental items read") assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(incBO.Errors.Failure())) @@ -1976,6 +2000,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveOwnerMigration() { // 2 on read/writes to account for metadata: 1 delta and 1 path. assert.LessOrEqual(t, 2, incBO.Results.ItemsWritten, "items written") + assert.LessOrEqual(t, 1, incBO.Results.NonMetaItemsWritten, "non meta items written") assert.LessOrEqual(t, 2, incBO.Results.ItemsRead, "items read") assert.NoError(t, incBO.Errors.Failure(), "non-recoverable error", clues.ToCore(incBO.Errors.Failure())) assert.Empty(t, incBO.Errors.Recovered(), "recoverable/iteration errors") diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 942581e38..13e9a530a 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -1198,6 +1198,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems mds := ssmock.Streamer{Deets: test.populatedDetails} w := &store.Wrapper{Storer: mockBackupStorer{entries: test.populatedModels}} deets := details.Builder{} + writeStats := kopia.BackupStats{} err := mergeDetails( ctx, @@ -1206,6 +1207,8 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems test.inputMans, test.mdm, &deets, + &writeStats, + path.OneDriveService, fault.New(true)) test.errCheck(t, err, clues.ToCore(err)) @@ -1307,9 +1310,10 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde defer flush() var ( - mds = ssmock.Streamer{Deets: populatedDetails} - w = &store.Wrapper{Storer: mockBackupStorer{entries: populatedModels}} - deets = details.Builder{} + mds = ssmock.Streamer{Deets: populatedDetails} + w = &store.Wrapper{Storer: mockBackupStorer{entries: populatedModels}} + deets = details.Builder{} + writeStats = kopia.BackupStats{} ) err := mergeDetails( @@ -1319,6 +1323,8 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde inputMans, mdm, &deets, + &writeStats, + path.ExchangeService, fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) compareDeetEntries(t, expectedEntries, deets.Details().Entries) diff --git a/src/internal/stats/stats.go b/src/internal/stats/stats.go index c061e67bc..88f54b992 100644 --- a/src/internal/stats/stats.go +++ b/src/internal/stats/stats.go @@ -8,11 +8,13 @@ import ( // ReadWrites tracks the total count of reads and writes. ItemsRead // and ItemsWritten counts are assumed to be successful reads. type ReadWrites struct { - BytesRead int64 `json:"bytesRead,omitempty"` - BytesUploaded int64 `json:"bytesUploaded,omitempty"` - ItemsRead int `json:"itemsRead,omitempty"` - ItemsWritten int `json:"itemsWritten,omitempty"` - ResourceOwners int `json:"resourceOwners,omitempty"` + BytesRead int64 `json:"bytesRead,omitempty"` + BytesUploaded int64 `json:"bytesUploaded,omitempty"` + ItemsRead int `json:"itemsRead,omitempty"` + NonMetaBytesUploaded int64 `json:"nonMetaBytesUploaded,omitempty"` + NonMetaItemsWritten int `json:"nonMetaItemsWritten,omitempty"` + ItemsWritten int `json:"itemsWritten,omitempty"` + ResourceOwners int `json:"resourceOwners,omitempty"` } // StartAndEndTime tracks a paired starting time and ending time. diff --git a/src/pkg/backup/backup.go b/src/pkg/backup/backup.go index 8580a041e..39e317513 100644 --- a/src/pkg/backup/backup.go +++ b/src/pkg/backup/backup.go @@ -284,12 +284,12 @@ func (b Backup) toStats() backupStats { return backupStats{ ID: string(b.ID), BytesRead: b.BytesRead, - BytesUploaded: b.BytesUploaded, + BytesUploaded: b.NonMetaBytesUploaded, EndedAt: b.CompletedAt, ErrorCount: b.ErrorCount, ItemsRead: b.ItemsRead, ItemsSkipped: b.TotalSkippedItems, - ItemsWritten: b.ItemsWritten, + ItemsWritten: b.NonMetaItemsWritten, StartedAt: b.StartedAt, } } diff --git a/src/pkg/backup/backup_test.go b/src/pkg/backup/backup_test.go index 4f26f35c8..6cb009b3a 100644 --- a/src/pkg/backup/backup_test.go +++ b/src/pkg/backup/backup_test.go @@ -49,10 +49,12 @@ func stubBackup(t time.Time, ownerID, ownerName string) backup.Backup { ErrorCount: 2, Failure: "read, write", ReadWrites: stats.ReadWrites{ - BytesRead: 301, - BytesUploaded: 301, - ItemsRead: 1, - ItemsWritten: 1, + BytesRead: 301, + BytesUploaded: 301, + NonMetaBytesUploaded: 301, + ItemsRead: 1, + NonMetaItemsWritten: 1, + ItemsWritten: 1, }, StartAndEndTime: stats.StartAndEndTime{ StartedAt: t, @@ -248,7 +250,7 @@ func (suite *BackupUnitSuite) TestBackup_MinimumPrintable() { assert.Equal(t, now, result.Stats.StartedAt, "started at") assert.Equal(t, b.Status, result.Status, "status") assert.Equal(t, b.BytesRead, result.Stats.BytesRead, "size") - assert.Equal(t, b.BytesUploaded, result.Stats.BytesUploaded, "stored size") + assert.Equal(t, b.NonMetaBytesUploaded, result.Stats.BytesUploaded, "stored size") assert.Equal(t, b.Selector.DiscreteOwner, result.Owner, "owner") } diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index 483594079..15a0e3a7d 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -240,12 +240,27 @@ func (dm DetailsModel) FilterMetaFiles() DetailsModel { return d2 } +// SumNonMetaFileSizes returns the total size of items excluding all the +// .meta files from the items. +func (dm DetailsModel) SumNonMetaFileSizes() int64 { + var size int64 + + // Items will provide only files and filter out folders + for _, ent := range dm.FilterMetaFiles().Items() { + size += ent.size() + } + + return size +} + // Check if a file is a metadata file. These are used to store // additional data like permissions (in case of Drive items) and are // not to be treated as regular files. func (de Entry) isMetaFile() bool { // sharepoint types not needed, since sharepoint permissions were // added after IsMeta was deprecated. + // Earlier onedrive backups used to store both metafiles and files in details. + // So filter out just the onedrive items and check for metafiles return de.ItemInfo.OneDrive != nil && de.ItemInfo.OneDrive.IsMeta } From 7cb01b1e2779370a26586102cc502d3d7a321f07 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 31 May 2023 11:27:15 -0700 Subject: [PATCH 02/10] Wrap new base finder and wire it up (#3524) Take the new base finder code, wrap it in a thin wrapper and use it in place of the previous way of finding base snapshots This solution is not focused on efficiency as it does result in looking up backup models multiple times. It's more to get the new way of finding bases out and then we can go back and smooth things over when we have the chance --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3202 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 1 + src/internal/kopia/base_finder.go | 37 +++++--- src/internal/kopia/base_finder_test.go | 24 ++--- src/internal/kopia/inject/inject.go | 2 +- src/internal/kopia/wrapper.go | 5 ++ src/internal/operations/backup.go | 23 ++++- src/internal/operations/backup_test.go | 36 ++++---- src/internal/operations/manifests.go | 39 ++++----- src/internal/operations/manifests_test.go | 102 +++++++++------------- 9 files changed, 141 insertions(+), 128 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50c391c64..5b0b9c711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added ProtectedResourceName to the backup list json output. ProtectedResourceName holds either a UPN or a WebURL, depending on the resource type. +- Rework base selection logic for incremental backups so it's more likely to find a valid base. ### Fixed - Fix Exchange folder cache population error when parent folder isn't found. diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index e1adef823..837691d89 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -15,10 +15,10 @@ import ( "github.com/alcionai/corso/src/pkg/logger" ) -type BackupBases struct { - Backups []BackupEntry - MergeBases []ManifestEntry - AssistBases []ManifestEntry +type backupBases struct { + backups []BackupEntry + mergeBases []ManifestEntry + assistBases []ManifestEntry } type BackupEntry struct { @@ -31,7 +31,7 @@ type baseFinder struct { bg inject.GetBackuper } -func NewBaseFinder( +func newBaseFinder( sm snapshotManager, bg inject.GetBackuper, ) (*baseFinder, error) { @@ -183,11 +183,11 @@ func (b *baseFinder) getBase( return b.findBasesInSet(ctx, reason, metas) } -func (b *baseFinder) FindBases( +func (b *baseFinder) findBases( ctx context.Context, reasons []Reason, tags map[string]string, -) (BackupBases, error) { +) (backupBases, error) { var ( // All maps go from ID -> entry. We need to track by ID so we can coalesce // the reason for selecting something. Kopia assisted snapshots also use @@ -251,9 +251,24 @@ func (b *baseFinder) FindBases( } } - return BackupBases{ - Backups: maps.Values(baseBups), - MergeBases: maps.Values(baseSnaps), - AssistBases: maps.Values(kopiaAssistSnaps), + return backupBases{ + backups: maps.Values(baseBups), + mergeBases: maps.Values(baseSnaps), + assistBases: maps.Values(kopiaAssistSnaps), }, nil } + +func (b *baseFinder) FindBases( + ctx context.Context, + reasons []Reason, + tags map[string]string, +) ([]ManifestEntry, error) { + bb, err := b.findBases(ctx, reasons, tags) + if err != nil { + return nil, clues.Stack(err) + } + + // assistBases contains all snapshots so we can return it while maintaining + // almost all compatibility. + return bb.assistBases, nil +} diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go index 4a9f47100..0c3761618 100644 --- a/src/internal/kopia/base_finder_test.go +++ b/src/internal/kopia/base_finder_test.go @@ -342,10 +342,10 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() { }, } - bb, err := bf.FindBases(ctx, reasons, nil) + bb, err := bf.findBases(ctx, reasons, nil) assert.NoError(t, err, "getting bases: %v", clues.ToCore(err)) - assert.Empty(t, bb.MergeBases) - assert.Empty(t, bb.AssistBases) + assert.Empty(t, bb.mergeBases) + assert.Empty(t, bb.assistBases) } func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { @@ -366,10 +366,10 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { }, } - bb, err := bf.FindBases(ctx, reasons, nil) + bb, err := bf.findBases(ctx, reasons, nil) assert.NoError(t, err, "getting bases: %v", clues.ToCore(err)) - assert.Empty(t, bb.MergeBases) - assert.Empty(t, bb.AssistBases) + assert.Empty(t, bb.mergeBases) + assert.Empty(t, bb.assistBases) } func (suite *BaseFinderUnitSuite) TestGetBases() { @@ -825,7 +825,7 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { bg: &mockModelGetter{data: test.backupData}, } - bb, err := bf.FindBases( + bb, err := bf.findBases( ctx, test.input, nil) @@ -833,17 +833,17 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { checkBackupEntriesMatch( t, - bb.Backups, + bb.backups, test.backupData, test.expectedBaseReasons) checkManifestEntriesMatch( t, - bb.MergeBases, + bb.mergeBases, test.manifestData, test.expectedBaseReasons) checkManifestEntriesMatch( t, - bb.AssistBases, + bb.assistBases, test.manifestData, test.expectedAssistManifestReasons) }) @@ -920,7 +920,7 @@ func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() { bg: &mockModelGetter{data: backupData}, } - bb, err := bf.FindBases( + bb, err := bf.findBases( ctx, testAllUsersAllCats, test.tags) @@ -928,7 +928,7 @@ func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() { checkManifestEntriesMatch( t, - bb.MergeBases, + bb.mergeBases, manifestData, test.expectedIdxs) }) diff --git a/src/internal/kopia/inject/inject.go b/src/internal/kopia/inject/inject.go index be1d9a16b..d97e06d31 100644 --- a/src/internal/kopia/inject/inject.go +++ b/src/internal/kopia/inject/inject.go @@ -39,6 +39,6 @@ type ( ctx context.Context, reasons []kopia.Reason, tags map[string]string, - ) (kopia.BackupBases, error) + ) ([]kopia.ManifestEntry, error) } ) diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index e92d2e133..f7830eded 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control/repository" @@ -616,6 +617,10 @@ func (w Wrapper) FetchPrevSnapshotManifests( return fetchPrevSnapshotManifests(ctx, w.c, reasons, tags), nil } +func (w Wrapper) NewBaseFinder(bg inject.GetBackuper) (*baseFinder, error) { + return newBaseFinder(w.c, bg) +} + func isErrEntryNotFound(err error) bool { // Calling Child on a directory may return this. if errors.Is(err, fs.ErrEntryNotFound) { diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 92e95c53b..5240d2c88 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -290,9 +290,24 @@ func (op *BackupOperation) do( // should always be 1, since backups are 1:1 with resourceOwners. opStats.resourceCount = 1 + kbf, err := op.kopia.NewBaseFinder(op.store) + if err != nil { + return nil, clues.Stack(err) + } + + type baseFinder struct { + kinject.BaseFinder + kinject.RestoreProducer + } + + bf := baseFinder{ + BaseFinder: kbf, + RestoreProducer: op.kopia, + } + mans, mdColls, canUseMetaData, err := produceManifestsAndMetadata( ctx, - op.kopia, + bf, op.store, reasons, fallbackReasons, op.account.ID(), @@ -467,7 +482,7 @@ func consumeBackupCollections( bc kinject.BackupConsumer, tenantID string, reasons []kopia.Reason, - mans []*kopia.ManifestEntry, + mans []kopia.ManifestEntry, cs []data.BackupCollection, pmr prefixmatcher.StringSetReader, backupID model.StableID, @@ -652,7 +667,7 @@ func getNewPathRefs( func lastCompleteBackups( ctx context.Context, ms *store.Wrapper, - mans []*kopia.ManifestEntry, + mans []kopia.ManifestEntry, ) (map[string]*backup.Backup, int, error) { var ( oldestVersion = version.NoBackup @@ -703,7 +718,7 @@ func mergeDetails( ctx context.Context, ms *store.Wrapper, detailsStore streamstore.Streamer, - mans []*kopia.ManifestEntry, + mans []kopia.ManifestEntry, dataFromBackup kopia.DetailsMergeInfoer, deets *details.Builder, writeStats *kopia.BackupStats, diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 13e9a530a..94f0b5c7a 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -533,12 +533,12 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections table := []struct { name string - inputMan []*kopia.ManifestEntry + inputMan []kopia.ManifestEntry expected []kopia.IncrementalBase }{ { name: "SingleManifestSingleReason", - inputMan: []*kopia.ManifestEntry{ + inputMan: []kopia.ManifestEntry{ { Manifest: manifest1, Reasons: []kopia.Reason{ @@ -557,7 +557,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections }, { name: "SingleManifestMultipleReasons", - inputMan: []*kopia.ManifestEntry{ + inputMan: []kopia.ManifestEntry{ { Manifest: manifest1, Reasons: []kopia.Reason{ @@ -578,7 +578,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections }, { name: "MultipleManifestsMultipleReasons", - inputMan: []*kopia.ManifestEntry{ + inputMan: []kopia.ManifestEntry{ { Manifest: manifest1, Reasons: []kopia.Reason{ @@ -731,7 +731,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems name string populatedModels map[model.StableID]backup.Backup populatedDetails map[string]*details.Details - inputMans []*kopia.ManifestEntry + inputMans []kopia.ManifestEntry mdm *mockDetailsMergeInfoer errCheck assert.ErrorAssertionFunc @@ -758,7 +758,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), "foo", ""), Reasons: []kopia.Reason{ @@ -776,7 +776,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -803,7 +803,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -833,7 +833,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -869,7 +869,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -931,7 +931,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -961,7 +961,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -994,7 +994,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -1027,7 +1027,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -1061,7 +1061,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -1095,7 +1095,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -1146,7 +1146,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems return res }(), - inputMans: []*kopia.ManifestEntry{ + inputMans: []kopia.ManifestEntry{ { Manifest: makeManifest(suite.T(), backup1.ID, ""), Reasons: []kopia.Reason{ @@ -1258,7 +1258,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde Category: itemPath1.Category(), } - inputMans = []*kopia.ManifestEntry{ + inputMans = []kopia.ManifestEntry{ { Manifest: makeManifest(t, backup1.ID, ""), Reasons: []kopia.Reason{ diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index 476b99126..2b028c55a 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -19,16 +19,8 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -type manifestFetcher interface { - FetchPrevSnapshotManifests( - ctx context.Context, - reasons []kopia.Reason, - tags map[string]string, - ) ([]*kopia.ManifestEntry, error) -} - type manifestRestorer interface { - manifestFetcher + inject.BaseFinder inject.RestoreProducer } @@ -47,14 +39,14 @@ func produceManifestsAndMetadata( reasons, fallbackReasons []kopia.Reason, tenantID string, getMetadata bool, -) ([]*kopia.ManifestEntry, []data.RestoreCollection, bool, error) { +) ([]kopia.ManifestEntry, []data.RestoreCollection, bool, error) { var ( tags = map[string]string{kopia.TagBackupCategory: ""} metadataFiles = graph.AllMetadataFileNames() collections []data.RestoreCollection ) - ms, err := mr.FetchPrevSnapshotManifests(ctx, reasons, tags) + ms, err := mr.FindBases(ctx, reasons, tags) if err != nil { return nil, nil, false, clues.Wrap(err, "looking up prior snapshots") } @@ -70,7 +62,7 @@ func produceManifestsAndMetadata( return ms, nil, false, nil } - fbms, err := mr.FetchPrevSnapshotManifests(ctx, fallbackReasons, tags) + fbms, err := mr.FindBases(ctx, fallbackReasons, tags) if err != nil { return nil, nil, false, clues.Wrap(err, "looking up prior snapshots under alternate id") } @@ -177,9 +169,9 @@ func produceManifestsAndMetadata( // 3. If mans has no entry for a reason, look for both complete and incomplete fallbacks. func unionManifests( reasons []kopia.Reason, - mans []*kopia.ManifestEntry, - fallback []*kopia.ManifestEntry, -) []*kopia.ManifestEntry { + mans []kopia.ManifestEntry, + fallback []kopia.ManifestEntry, +) []kopia.ManifestEntry { if len(fallback) == 0 { return mans } @@ -203,7 +195,9 @@ func unionManifests( } // track the manifests that were collected with the current lookup - for _, m := range mans { + for i := range mans { + m := &mans[i] + for _, r := range m.Reasons { k := r.Service.String() + r.Category.String() t := tups[k] @@ -219,7 +213,8 @@ func unionManifests( } // backfill from the fallback where necessary - for _, m := range fallback { + for i := range fallback { + m := &fallback[i] useReasons := []kopia.Reason{} for _, r := range m.Reasons { @@ -250,15 +245,15 @@ func unionManifests( } // collect the results into a single slice of manifests - ms := map[string]*kopia.ManifestEntry{} + ms := map[string]kopia.ManifestEntry{} for _, m := range tups { if m.complete != nil { - ms[string(m.complete.ID)] = m.complete + ms[string(m.complete.ID)] = *m.complete } if m.incomplete != nil { - ms[string(m.incomplete.ID)] = m.incomplete + ms[string(m.incomplete.ID)] = *m.incomplete } } @@ -269,7 +264,7 @@ func unionManifests( // of manifests, that each manifest's Reason (owner, service, category) is only // included once. If a reason is duplicated by any two manifests, an error is // returned. -func verifyDistinctBases(ctx context.Context, mans []*kopia.ManifestEntry) error { +func verifyDistinctBases(ctx context.Context, mans []kopia.ManifestEntry) error { reasons := map[string]manifest.ID{} for _, man := range mans { @@ -303,7 +298,7 @@ func verifyDistinctBases(ctx context.Context, mans []*kopia.ManifestEntry) error func collectMetadata( ctx context.Context, r inject.RestoreProducer, - man *kopia.ManifestEntry, + man kopia.ManifestEntry, fileNames []string, tenantID string, errs *fault.Bus, diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index fa3e93a6a..4c359edb1 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -27,16 +27,16 @@ import ( type mockManifestRestorer struct { mockRestoreProducer - mans []*kopia.ManifestEntry + mans []kopia.ManifestEntry mrErr error // err varname already claimed by mockRestoreProducer } -func (mmr mockManifestRestorer) FetchPrevSnapshotManifests( +func (mmr mockManifestRestorer) FindBases( ctx context.Context, reasons []kopia.Reason, tags map[string]string, -) ([]*kopia.ManifestEntry, error) { - mans := map[string]*kopia.ManifestEntry{} +) ([]kopia.ManifestEntry, error) { + mans := map[string]kopia.ManifestEntry{} for _, r := range reasons { for _, m := range mmr.mans { @@ -49,10 +49,6 @@ func (mmr mockManifestRestorer) FetchPrevSnapshotManifests( } } - if len(mans) == 0 && len(reasons) == 0 { - return mmr.mans, mmr.mrErr - } - return maps.Values(mans), mmr.mrErr } @@ -247,7 +243,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { mr := mockRestoreProducer{err: test.expectErr} mr.buildRestoreFunc(t, test.manID, paths) - man := &kopia.ManifestEntry{ + man := kopia.ManifestEntry{ Manifest: &snapshot.Manifest{ID: manifest.ID(test.manID)}, Reasons: test.reasons, } @@ -263,12 +259,12 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { table := []struct { name string - mans []*kopia.ManifestEntry + mans []kopia.ManifestEntry expect assert.ErrorAssertionFunc }{ { name: "one manifest, one reason", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ @@ -284,7 +280,7 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { }, { name: "one incomplete manifest", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{IncompleteReason: "ir"}, }, @@ -293,7 +289,7 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { }, { name: "one manifest, multiple reasons", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ @@ -314,7 +310,7 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { }, { name: "one manifest, duplicate reasons", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ @@ -335,7 +331,7 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { }, { name: "two manifests, non-overlapping reasons", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ @@ -361,7 +357,7 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { }, { name: "two manifests, overlapping reasons", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ @@ -387,7 +383,7 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() { }, { name: "two manifests, overlapping reasons, one snapshot incomplete", - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ @@ -430,13 +426,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { did = "detailsid" ) - makeMan := func(pct path.CategoryType, id, incmpl, bid string) *kopia.ManifestEntry { + makeMan := func(pct path.CategoryType, id, incmpl, bid string) kopia.ManifestEntry { tags := map[string]string{} if len(bid) > 0 { tags = map[string]string{"tag:" + kopia.TagBackupID: bid} } - return &kopia.ManifestEntry{ + return kopia.ManifestEntry{ Manifest: &snapshot.Manifest{ ID: manifest.ID(id), IncompleteReason: incmpl, @@ -456,7 +452,6 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name string mr mockManifestRestorer gb mockGetBackuper - reasons []kopia.Reason getMeta bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc @@ -467,10 +462,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "don't get metadata, no mans", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{}, + mans: []kopia.ManifestEntry{}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -480,10 +474,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "don't get metadata", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "", "")}, + mans: []kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "", "")}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -493,10 +486,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "don't get metadata, incomplete manifest", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "ir", "")}, + mans: []kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "ir", "")}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -509,7 +501,6 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { mrErr: assert.AnError, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.Error, assertB: assert.False, @@ -519,13 +510,12 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "verify distinct bases fails", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ makeMan(path.EmailCategory, "id1", "", ""), makeMan(path.EmailCategory, "id2", "", ""), }, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, // No error, even though verify failed. assertB: assert.False, @@ -535,10 +525,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "no manifests", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{}, + mans: []kopia.ManifestEntry{}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -548,13 +537,12 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "only incomplete manifests", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ makeMan(path.EmailCategory, "id1", "ir", ""), makeMan(path.ContactsCategory, "id2", "ir", ""), }, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -568,10 +556,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id": {data.NotFoundRestoreCollection{Collection: mockColl{id: "id_coll"}}}, }, }, - mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id", "", "")}, + mans: []kopia.ManifestEntry{makeMan(path.EmailCategory, "id", "", "")}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.Error, assertB: assert.False, @@ -581,10 +568,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "backup missing details id", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{}, - mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "", "bid")}, + mans: []kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "", "bid")}, }, gb: mockGetBackuper{}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, assertB: assert.False, @@ -598,13 +584,12 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "incmpl_id": {data.NotFoundRestoreCollection{Collection: mockColl{id: "incmpl_id_coll"}}}, }, }, - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ makeMan(path.EmailCategory, "id", "", "bid"), makeMan(path.EmailCategory, "incmpl_id", "ir", ""), }, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -618,10 +603,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id": {data.NotFoundRestoreCollection{Collection: mockColl{id: "id_coll"}}}, }, }, - mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id", "", "bid")}, + mans: []kopia.ManifestEntry{makeMan(path.EmailCategory, "id", "", "bid")}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -636,13 +620,12 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "contact": {data.NotFoundRestoreCollection{Collection: mockColl{id: "contact_coll"}}}, }, }, - mans: []*kopia.ManifestEntry{ + mans: []kopia.ManifestEntry{ makeMan(path.EmailCategory, "mail", "", "bid"), makeMan(path.ContactsCategory, "contact", "", "bid"), }, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -655,10 +638,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name: "error collecting metadata", mr: mockManifestRestorer{ mockRestoreProducer: mockRestoreProducer{err: assert.AnError}, - mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "", "bid")}, + mans: []kopia.ManifestEntry{makeMan(path.EmailCategory, "id1", "", "bid")}, }, gb: mockGetBackuper{detailsID: did}, - reasons: []kopia.Reason{}, getMeta: true, assertErr: assert.Error, assertB: assert.False, @@ -677,7 +659,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { ctx, &test.mr, &test.gb, - test.reasons, nil, + []kopia.Reason{{ResourceOwner: ro}}, nil, tid, test.getMeta) test.assertErr(t, err, clues.ToCore(err)) @@ -739,8 +721,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb fbIncomplete = "fb_incmpl" ) - makeMan := func(id, incmpl string, reasons []kopia.Reason) *kopia.ManifestEntry { - return &kopia.ManifestEntry{ + makeMan := func(id, incmpl string, reasons []kopia.Reason) kopia.ManifestEntry { + return kopia.ManifestEntry{ Manifest: &snapshot.Manifest{ ID: manifest.ID(id), IncompleteReason: incmpl, @@ -1005,7 +987,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb }) } - mans := []*kopia.ManifestEntry{} + mans := []kopia.ManifestEntry{} for _, m := range test.man { incomplete := "" @@ -1027,7 +1009,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb mr := mockManifestRestorer{mans: mans} - mans, _, b, err := produceManifestsAndMetadata( + gotMans, _, b, err := produceManifestsAndMetadata( ctx, &mr, nil, @@ -1040,7 +1022,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb manIDs := []string{} - for _, m := range mans { + for _, m := range gotMans { manIDs = append(manIDs, string(m.ID)) reasons := test.expectReasons[string(m.ID)] @@ -1075,12 +1057,12 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_VerifyDistinctBases() table := []struct { name string - input []*kopia.ManifestEntry + input []kopia.ManifestEntry errCheck assert.ErrorAssertionFunc }{ { name: "SingleManifestMultipleReasons", - input: []*kopia.ManifestEntry{ + input: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{ ID: "id1", @@ -1103,7 +1085,7 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_VerifyDistinctBases() }, { name: "MultipleManifestsDistinctReason", - input: []*kopia.ManifestEntry{ + input: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{ ID: "id1", @@ -1133,7 +1115,7 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_VerifyDistinctBases() }, { name: "MultipleManifestsSameReason", - input: []*kopia.ManifestEntry{ + input: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{ ID: "id1", @@ -1163,7 +1145,7 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_VerifyDistinctBases() }, { name: "MultipleManifestsSameReasonOneIncomplete", - input: []*kopia.ManifestEntry{ + input: []kopia.ManifestEntry{ { Manifest: &snapshot.Manifest{ ID: "id1", @@ -1250,13 +1232,13 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_CollectMetadata() { table := []struct { name string - inputMan *kopia.ManifestEntry + inputMan kopia.ManifestEntry inputFiles []string expected []path.Path }{ { name: "SingleReasonSingleFile", - inputMan: &kopia.ManifestEntry{ + inputMan: kopia.ManifestEntry{ Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ { @@ -1271,7 +1253,7 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_CollectMetadata() { }, { name: "SingleReasonMultipleFiles", - inputMan: &kopia.ManifestEntry{ + inputMan: kopia.ManifestEntry{ Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ { @@ -1286,7 +1268,7 @@ func (suite *BackupManifestUnitSuite) TestBackupOperation_CollectMetadata() { }, { name: "MultipleReasonsMultipleFiles", - inputMan: &kopia.ManifestEntry{ + inputMan: kopia.ManifestEntry{ Manifest: &snapshot.Manifest{}, Reasons: []kopia.Reason{ { From 27d58892935480d30630f132e4302357ee34c44a Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 31 May 2023 15:29:43 -0700 Subject: [PATCH 03/10] Don't try to delete test data in the test (#3547) Data should be cleaned up by purge script when it runs so there's no need to risk test failures by trying to clean it up at the end of the test --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/connector/exchange/restore_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index 78892f361..5c60dbbb6 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -151,10 +151,6 @@ func (suite *RestoreIntgSuite) TestRestoreEvent() { } } -type containerDeleter interface { - DeleteContainer(context.Context, string, string) error -} - // TestRestoreExchangeObject verifies path.Category usage for restored objects func (suite *RestoreIntgSuite) TestRestoreExchangeObject() { t := suite.T() @@ -165,12 +161,6 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() { service, err := createService(m365) require.NoError(t, err, clues.ToCore(err)) - deleters := map[path.CategoryType]containerDeleter{ - path.EmailCategory: suite.ac.Mail(), - path.ContactsCategory: suite.ac.Contacts(), - path.EventsCategory: suite.ac.Events(), - } - userID := tester.M365UserID(suite.T()) tests := []struct { @@ -379,10 +369,6 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() { fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) assert.NotNil(t, info, "item info was not populated") - assert.NotNil(t, deleters) - - err = deleters[test.category].DeleteContainer(ctx, userID, destination) - assert.NoError(t, err, clues.ToCore(err)) }) } } From 8dc829eebe234fabf02d16587c729ec141302738 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 31 May 2023 18:48:04 -0600 Subject: [PATCH 04/10] remove the golang purge script (#3544) fully replaced by powershell scripts --- src/cmd/purge/purge.go | 325 ----------------------------------------- 1 file changed, 325 deletions(-) delete mode 100644 src/cmd/purge/purge.go diff --git a/src/cmd/purge/purge.go b/src/cmd/purge/purge.go deleted file mode 100644 index 70948a2b3..000000000 --- a/src/cmd/purge/purge.go +++ /dev/null @@ -1,325 +0,0 @@ -package main - -import ( - "context" - "os" - "time" - - "github.com/alcionai/clues" - "github.com/pkg/errors" - "github.com/spf13/cobra" - - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/internal/connector" - "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/credentials" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/logger" - "github.com/alcionai/corso/src/pkg/services/m365" - "github.com/alcionai/corso/src/pkg/services/m365/api" -) - -var purgeCmd = &cobra.Command{ - Use: "purge", - Short: "Purge all types of m365 folders", - RunE: handleAllFolderPurge, -} - -var oneDriveCmd = &cobra.Command{ - Use: "onedrive", - Short: "Purges OneDrive folders", - RunE: handleOneDriveFolderPurge, -} - -var ( - before string - user string - tenant string - prefix string -) - -var ErrPurging = clues.New("not all items were successfully purged") - -// ------------------------------------------------------------------------------------------ -// CLI command handlers -// ------------------------------------------------------------------------------------------ - -func main() { - ls := logger.Settings{ - Level: logger.LLDebug, - Format: logger.LFText, - } - ctx, _ := logger.CtxOrSeed(context.Background(), ls) - ctx = SetRootCmd(ctx, purgeCmd) - - defer logger.Flush(ctx) - - fs := purgeCmd.PersistentFlags() - fs.StringVar(&before, "before", "", "folders older than this date are deleted. (default: now in UTC)") - fs.StringVar(&user, "user", "", "m365 user id whose folders will be deleted") - cobra.CheckErr(purgeCmd.MarkPersistentFlagRequired("user")) - fs.StringVar(&tenant, "tenant", "", "m365 tenant containing the user") - fs.StringVar(&prefix, "prefix", "", "filters mail folders by displayName prefix") - cobra.CheckErr(purgeCmd.MarkPersistentFlagRequired("prefix")) - - purgeCmd.AddCommand(oneDriveCmd) - - if err := purgeCmd.ExecuteContext(ctx); err != nil { - logger.Flush(ctx) - os.Exit(1) - } -} - -func handleAllFolderPurge(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - - if utils.HasNoFlagsAndShownHelp(cmd) { - return nil - } - - acct, gc, t, err := getGCAndBoundaryTime(ctx) - if err != nil { - return err - } - - err = runPurgeForEachUser( - ctx, - acct, - gc, - t, - purgeOneDriveFolders, - ) - if err != nil { - return Only(ctx, ErrPurging) - } - - return nil -} - -func handleOneDriveFolderPurge(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - - if utils.HasNoFlagsAndShownHelp(cmd) { - return nil - } - - acct, gc, t, err := getGCAndBoundaryTime(ctx) - if err != nil { - return err - } - - if err := runPurgeForEachUser(ctx, acct, gc, t, purgeOneDriveFolders); err != nil { - logger.Ctx(ctx).Error(err) - return Only(ctx, clues.Wrap(ErrPurging, "OneDrive folders")) - } - - return nil -} - -// ------------------------------------------------------------------------------------------ -// Purge Controllers -// ------------------------------------------------------------------------------------------ - -type purgable interface { - GetDisplayName() *string - GetId() *string -} - -type purger func(context.Context, *connector.GraphConnector, time.Time, string) error - -func runPurgeForEachUser( - ctx context.Context, - acct account.Account, - gc *connector.GraphConnector, - boundary time.Time, - ps ...purger, -) error { - users, err := m365.Users(ctx, acct, fault.New(true)) - if err != nil { - return clues.Wrap(err, "getting users") - } - - for _, u := range userOrUsers(user, users) { - Infof(ctx, "\nUser: %s - %s", u.PrincipalName, u.ID) - - for _, p := range ps { - if err := p(ctx, gc, boundary, u.PrincipalName); err != nil { - return err - } - } - } - - return nil -} - -// ----- OneDrive - -func purgeOneDriveFolders( - ctx context.Context, - gc *connector.GraphConnector, - boundary time.Time, - uid string, -) error { - getter := func(gs graph.Servicer, uid, prefix string) ([]purgable, error) { - pager, err := onedrive.PagerForSource(onedrive.OneDriveSource, gs, uid, nil) - if err != nil { - return nil, err - } - - cfs, err := onedrive.GetAllFolders(ctx, gs, pager, prefix, fault.New(true)) - if err != nil { - return nil, err - } - - purgables := make([]purgable, len(cfs)) - - for i, v := range cfs { - purgables[i] = v - } - - return purgables, nil - } - - deleter := func(gs graph.Servicer, uid string, f purgable) error { - driveFolder, ok := f.(*onedrive.Displayable) - if !ok { - return clues.New("non-OneDrive item") - } - - return api.DeleteDriveItem( - ctx, - gs, - *driveFolder.GetParentReference().GetDriveId(), - *f.GetId()) - } - - return purgeFolders(ctx, gc, boundary, "OneDrive Folders", uid, getter, deleter) -} - -// ----- controller - -func purgeFolders( - ctx context.Context, - gc *connector.GraphConnector, - boundary time.Time, - data, uid string, - getter func(graph.Servicer, string, string) ([]purgable, error), - deleter func(graph.Servicer, string, purgable) error, -) error { - Infof(ctx, "Container: %s", data) - - // get them folders - fs, err := getter(gc.Service, uid, prefix) - if err != nil { - return Only(ctx, clues.Wrap(err, "retrieving folders: "+data)) - } - - if len(fs) == 0 { - Info(ctx, "None Matched") - return nil - } - - var errs error - - // delete any containers that don't pass the boundary - for _, fld := range fs { - // compare the folder time to the deletion boundary time first - displayName := *fld.GetDisplayName() - - dnTime, err := dttm.ExtractTime(displayName) - if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { - err = clues.Wrap(err, "!! Error: parsing container: "+displayName) - Info(ctx, err) - - return err - } - - if !dnTime.Before(boundary) || dnTime == (time.Time{}) { - continue - } - - Infof(ctx, "∙ Deleting [%s]", displayName) - - err = deleter(gc.Service, uid, fld) - if err != nil { - err = clues.Wrap(err, "!! Error") - Info(ctx, err) - } - } - - return errs -} - -// ------------------------------------------------------------------------------------------ -// Helpers -// ------------------------------------------------------------------------------------------ - -func getGC(ctx context.Context) (account.Account, *connector.GraphConnector, error) { - // get account info - m365Cfg := account.M365Config{ - M365: credentials.GetM365(), - AzureTenantID: str.First(tenant, os.Getenv(account.AzureTenantID)), - } - - acct, err := account.NewAccount(account.ProviderM365, m365Cfg) - if err != nil { - return account.Account{}, nil, Only(ctx, clues.Wrap(err, "finding m365 account details")) - } - - gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) - if err != nil { - return account.Account{}, nil, Only(ctx, clues.Wrap(err, "connecting to graph api")) - } - - return acct, gc, nil -} - -func getBoundaryTime(ctx context.Context) (time.Time, error) { - // format the time input - var ( - err error - boundaryTime = time.Now().UTC() - ) - - if len(before) > 0 { - boundaryTime, err = dttm.ParseTime(before) - if err != nil { - return time.Time{}, Only(ctx, clues.Wrap(err, "parsing before flag to time")) - } - } - - return boundaryTime, nil -} - -func getGCAndBoundaryTime( - ctx context.Context, -) (account.Account, *connector.GraphConnector, time.Time, error) { - acct, gc, err := getGC(ctx) - if err != nil { - return account.Account{}, nil, time.Time{}, err - } - - t, err := getBoundaryTime(ctx) - if err != nil { - return account.Account{}, nil, time.Time{}, err - } - - return acct, gc, t, nil -} - -func userOrUsers(u string, us []*m365.User) []*m365.User { - if len(u) == 0 { - return nil - } - - if u == "*" { - return us - } - - return []*m365.User{{PrincipalName: u}} -} From e7f0d82d5d8c4eb318a7029f2076ea7e76d82fc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 04:06:02 +0000 Subject: [PATCH 05/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.271=20to=201.44.272=20in=20/src=20(#?= =?UTF-8?q?3540)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.271 to 1.44.272.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.272 (2023-05-30)

Service Client Updates

  • service/chime-sdk-voice: Updates service API and documentation
  • service/glue: Updates service API and documentation
    • Added Runtime parameter to allow selection of Ray Runtime
  • service/groundstation: Updates service documentation
  • service/iotfleetwise: Updates service API and documentation
  • service/location: Updates service API and documentation
  • service/memorydb: Updates service API
  • service/personalize: Updates service API and documentation
  • service/polly: Updates service API
    • Amazon Polly adds 2 new voices - Sofie (da-DK) and Niamh (en-IE)
  • service/securityhub: Updates service API, documentation, and examples
  • service/securitylake: Updates service API, documentation, and paginators
  • service/wafv2: Updates service API and documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.271&new-version=1.44.272)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 095b097a5..5c69c2af8 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.271 + github.com/aws/aws-sdk-go v1.44.272 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 476156075..effff1702 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.271 h1:aa+Nu2JcnFmW1TLIz/67SS7KPq1I1Adl4RmExSMjGVo= -github.com/aws/aws-sdk-go v1.44.271/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.272 h1:cP7BaZCz1a4P80vOxi8OVma8ZcxZNDYyuyU5/bojU+Y= +github.com/aws/aws-sdk-go v1.44.272/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 4b932057de14d513bb83676431d7ef12c3995074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 04:35:51 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/st?= =?UTF-8?q?retchr/testify=20from=201.8.3=20to=201.8.4=20in=20/src=20(#3539?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.3 to 1.8.4.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/stretchr/testify&package-manager=go_modules&previous-version=1.8.3&new-version=1.8.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 5c69c2af8..d21ebd2f4 100644 --- a/src/go.mod +++ b/src/go.mod @@ -27,7 +27,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.8.4 github.com/tidwall/pretty v1.2.1 github.com/tomlazar/table v0.1.2 github.com/vbauerster/mpb/v8 v8.1.6 diff --git a/src/go.sum b/src/go.sum index effff1702..e7f374ec6 100644 --- a/src/go.sum +++ b/src/go.sum @@ -387,8 +387,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU= From e46e75b7f212a31bbae85e979b4b1c0eb5e57d46 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 1 Jun 2023 00:06:44 -0600 Subject: [PATCH 07/10] updating gitignore to catch log files (#3549) Co-authored-by: aviator-app[bot] <48659329+aviator-app[bot]@users.noreply.github.com> --- .gitignore | 9 ++++----- src/.gitignore | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 3f0ba69bf..bdda0724b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ # Test binary, built with `go test -c` *.test +test_results/ +testlog/ # Output of the go coverage tool, specifically when used with LiteIDE *.out @@ -21,12 +23,9 @@ .corso.toml # Logging -.corso.log +*.log # Build directories /bin /docker/bin -/website/dist - -*/test_results/** -*/testlog/** \ No newline at end of file +/website/dist \ No newline at end of file diff --git a/src/.gitignore b/src/.gitignore index bca1e9bf2..b06b4dde6 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1,2 +1,26 @@ dist/ corso + +# Test binary, built with `go test -c` +*.test +test_results/ +testlog/ + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# IDE +.vscode +*.swp + +# Standard configuration file names +.corso_test.toml +.corso.toml + +# Logging +*.log + +# Build directories +/bin +/docker/bin +/website/dist From 59ff00ceaace76b147d5b35bd9e40e1740485645 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 06:31:25 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/sp?= =?UTF-8?q?f13/viper=20from=201.15.0=20to=201.16.0=20in=20/src=20(#3538)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.15.0 to 1.16.0.
Release notes

Sourced from github.com/spf13/viper's releases.

v1.16.0

What's Changed

Enhancements 🚀

Bug Fixes 🐛

Dependency Updates ⬆️

Other Changes

New Contributors

Full Changelog: https://github.com/spf13/viper/compare/v1.15.0...v1.16.0

Commits
  • 21a7fd8 Allow ConfigParseError to unwrap (#1433)
  • 53cdb52 chore: improve log messages
  • 24ece16 drop std' logger in favor of jww one
  • 98b1b9f add:viper: allow to disable internal log messages
  • 2ee1631 extend viper's pflag binding to stringToInt pflag
  • 8652057 docs: Coder uses Viper
  • b850f34 build(deps): bump github.com/pelletier/go-toml/v2 from 2.0.7 to 2.0.8
  • 23b123a doc: add bearer/bearer as projects using Viper
  • a0fed9f Update main image to better handle dark background
  • 143e3bc build(deps): bump github.com/stretchr/testify from 1.8.2 to 1.8.3
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/viper&package-manager=go_modules&previous-version=1.15.0&new-version=1.16.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 12 ++++++------ src/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/go.mod b/src/go.mod index d21ebd2f4..4edb223a0 100644 --- a/src/go.mod +++ b/src/go.mod @@ -26,7 +26,7 @@ require ( github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.15.0 + github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/pretty v1.2.1 github.com/tomlazar/table v0.1.2 @@ -49,9 +49,9 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.6 // indirect - github.com/spf13/afero v1.9.3 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -115,14 +115,14 @@ require ( go.opentelemetry.io/otel/trace v1.15.1 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.8.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.54.0 // indirect + google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/src/go.sum b/src/go.sum index e7f374ec6..9423688ad 100644 --- a/src/go.sum +++ b/src/go.sum @@ -113,7 +113,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -312,8 +312,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -349,7 +349,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY= @@ -363,18 +363,18 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 h1:lQ3JvmcVO1/AMFbabvUSJ4YtJRpEAX9Qza73p5j03sw= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1/go.mod h1:4aKqcbhASNqjbrG0h9BmkzcWvPJGxbef4B+j0XfFrZo= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= -github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -386,7 +386,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= @@ -450,10 +450,10 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -758,8 +758,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 2da554f00e0a9eabf844a485c94eaa45d42feae8 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 1 Jun 2023 01:01:08 -0600 Subject: [PATCH 09/10] minor logging updates (#3546) Co-authored-by: aviator-app[bot] <48659329+aviator-app[bot]@users.noreply.github.com> --- src/internal/connector/graph/middleware.go | 51 ++++++++-------------- src/internal/operations/backup.go | 3 +- src/internal/operations/restore.go | 8 +++- 3 files changed, 26 insertions(+), 36 deletions(-) diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index 03d3a109f..bc9aabe2d 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -140,29 +140,21 @@ func (mw *LoggingMiddleware) Intercept( var ( log = logger.Ctx(ctx) respClass = resp.StatusCode / 100 - logExtra = logger.DebugAPIFV || os.Getenv(logGraphRequestsEnvKey) != "" + + // special cases where we always dump the response body, since the response + // details might be critical to understanding the response when debugging. + // * 400-bad-request + // * 403-forbidden + logBody = logger.DebugAPIFV || + os.Getenv(logGraphRequestsEnvKey) != "" || + resp.StatusCode == http.StatusBadRequest || + resp.StatusCode == http.StatusForbidden ) - // special case: always info log 429 responses + // special case: always info-level status 429 logs if resp.StatusCode == http.StatusTooManyRequests { - if logExtra { - log = log.With("response", getRespDump(ctx, resp, true)) - } - - log.Infow( - "graph api throttling", - "limit", resp.Header.Get(rateLimitHeader), - "remaining", resp.Header.Get(rateRemainingHeader), - "reset", resp.Header.Get(rateResetHeader), - "retry-after", resp.Header.Get(retryAfterHeader)) - - return resp, err - } - - // special case: always dump status-400-bad-request - if resp.StatusCode == http.StatusBadRequest { - log.With("response", getRespDump(ctx, resp, true)). - Error("graph api error: " + resp.Status) + log.With("response", getRespDump(ctx, resp, logBody)). + Info("graph api throttling") return resp, err } @@ -170,25 +162,18 @@ func (mw *LoggingMiddleware) Intercept( // Log api calls according to api debugging configurations. switch respClass { case 2: - if logExtra { + if logBody { // only dump the body if it's under a size limit. We don't want to copy gigs into memory for a log. dump := getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != "" && resp.ContentLength < logMBLimit) log.Infow("2xx graph api resp", "response", dump) } case 3: - log = log.With("redirect_location", LoggableURL(resp.Header.Get(locationHeader))) - - if logExtra { - log = log.With("response", getRespDump(ctx, resp, false)) - } - - log.Info("graph api redirect: " + resp.Status) + log.With("redirect_location", LoggableURL(resp.Header.Get(locationHeader))). + With("response", getRespDump(ctx, resp, false)). + Info("graph api redirect: " + resp.Status) default: - if logExtra { - log = log.With("response", getRespDump(ctx, resp, true)) - } - - log.Error("graph api error: " + resp.Status) + log.With("response", getRespDump(ctx, resp, logBody)). + Error("graph api error: " + resp.Status) } return resp, err diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 5240d2c88..5ad400bb3 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -177,7 +177,8 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { ctx = clues.Add( ctx, "tenant_id", clues.Hide(op.account.ID()), - "resource_owner", clues.Hide(op.ResourceOwner.Name()), + "resource_owner_id", op.ResourceOwner.ID(), + "resource_owner_name", clues.Hide(op.ResourceOwner.Name()), "backup_id", op.Results.BackupID, "service", op.Selectors.Service, "incremental", op.incremental) diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 55103cec7..5b2c9bd71 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -194,6 +194,10 @@ func (op *RestoreOperation) do( detailsStore streamstore.Reader, start time.Time, ) (*details.Details, error) { + logger.Ctx(ctx). + With("control_options", op.Options, "selectors", op.Selectors). + Info("restoring selection") + bup, deets, err := getBackupAndDetailsFromID( ctx, op.BackupID, @@ -213,7 +217,8 @@ func (op *RestoreOperation) do( ctx = clues.Add( ctx, - "resource_owner", bup.Selector.DiscreteOwner, + "resource_owner_id", bup.Selector.ID(), + "resource_owner_name", clues.Hide(bup.Selector.Name()), "details_entries", len(deets.Entries), "details_paths", len(paths), "backup_snapshot_id", bup.SnapshotID, @@ -230,7 +235,6 @@ func (op *RestoreOperation) do( }) observe.Message(ctx, fmt.Sprintf("Discovered %d items in backup %s to restore", len(paths), op.BackupID)) - logger.Ctx(ctx).With("control_options", op.Options, "selectors", op.Selectors).Info("restoring selection") kopiaComplete, closer := observe.MessageWithCompletion(ctx, "Enumerating items in repository") defer closer() From ff66d5c041f1272e95ec4d10f84414c17f2f0ffd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 08:43:45 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.272=20to=201.44.273=20in=20/src=20(#?= =?UTF-8?q?3552)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.272 to 1.44.273.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.273 (2023-05-31)

Service Client Updates

  • service/config: Updates service API and documentation
  • service/frauddetector: Updates service API and documentation
  • service/healthlake: Updates service API and documentation
  • service/m2: Updates service API and documentation
  • service/rds: Updates service API, documentation, waiters, paginators, and examples
    • This release adds support for changing the engine for Oracle using the ModifyDbInstance API
  • service/servicecatalog: Updates service documentation
    • Documentation updates for ServiceCatalog.
  • service/workspaces-web: Updates service API, documentation, and paginators
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.272&new-version=1.44.273)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 4edb223a0..2dd8fccae 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.272 + github.com/aws/aws-sdk-go v1.44.273 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 9423688ad..84fa701e3 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.272 h1:cP7BaZCz1a4P80vOxi8OVma8ZcxZNDYyuyU5/bojU+Y= -github.com/aws/aws-sdk-go v1.44.272/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.273 h1:CX8O0gK+cGrgUyv7bgJ6QQP9mQg7u5mweHdNzULH47c= +github.com/aws/aws-sdk-go v1.44.273/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=