From 99edf7d5b0a3393dd2824b28ca24118d52d6f8ed Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:48:57 -0700 Subject: [PATCH] Add and populate mod time for BaseModel (#4065) Get the last time a model was modified and return it in BaseModel. This will help with discovering what items can be garbage collected during incomplete backup cleanup as we don't want to accidentally delete in-flight backups. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3217 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/model_store.go | 1 + src/internal/kopia/model_store_test.go | 48 +++++++++++++++++++++----- src/internal/model/model.go | 5 ++- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/internal/kopia/model_store.go b/src/internal/kopia/model_store.go index 93ef6c182..b5e572844 100644 --- a/src/internal/kopia/model_store.go +++ b/src/internal/kopia/model_store.go @@ -210,6 +210,7 @@ func (ms ModelStore) populateBaseModelFromMetadata( base.ID = model.StableID(id) base.ModelVersion = v base.Tags = m.Labels + base.ModTime = m.ModTime stripHiddenTags(base.Tags) diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index 048817a54..0afc72a7c 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -4,6 +4,7 @@ import ( "context" "sync" "testing" + "time" "github.com/alcionai/clues" "github.com/google/uuid" @@ -34,6 +35,18 @@ func getModelStore(t *testing.T, ctx context.Context) *ModelStore { return &ModelStore{c: c, modelVersion: globalModelVersion} } +func assertEqualNoModTime(t *testing.T, expected, got *fooModel) { + t.Helper() + + expectedClean := *expected + gotClean := *got + + expectedClean.ModTime = time.Time{} + gotClean.ModTime = time.Time{} + + assert.Equal(t, expectedClean, gotClean) +} + // --------------- // unit tests // --------------- @@ -259,6 +272,8 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet() { // Avoid some silly test errors from comparing nil to empty map. foo.Tags = map[string]string{} + startTime := time.Now() + err := suite.m.Put(suite.ctx, test.s, foo) test.check(t, err, clues.ToCore(err)) @@ -273,11 +288,17 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet() { returned := &fooModel{} err = suite.m.Get(suite.ctx, test.s, foo.ID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + + assertEqualNoModTime(t, foo, returned) + assert.WithinDuration(t, startTime, returned.ModTime, 5*time.Second) + + returned = &fooModel{} err = suite.m.GetWithModelStoreID(suite.ctx, test.s, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + + assertEqualNoModTime(t, foo, returned) + assert.WithinDuration(t, startTime, returned.ModTime, 5*time.Second) }) } } @@ -324,11 +345,11 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet_PreSetID() { err = suite.m.Get(suite.ctx, mdl, foo.ID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) err = suite.m.GetWithModelStoreID(suite.ctx, mdl, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) }) } } @@ -350,11 +371,11 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet_WithTags() { returned := &fooModel{} err = suite.m.Get(suite.ctx, theModelType, foo.ID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) err = suite.m.GetWithModelStoreID(suite.ctx, theModelType, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) } func (suite *ModelStoreIntegrationSuite) TestGet_NotFoundErrors() { @@ -559,7 +580,16 @@ func (suite *ModelStoreIntegrationSuite) TestGetOfTypeWithTags() { ids, err := suite.m.GetIDsForType(suite.ctx, test.s, test.tags) require.NoError(t, err, clues.ToCore(err)) - assert.ElementsMatch(t, expected, ids) + cleanIDs := make([]*model.BaseModel, 0, len(ids)) + + for _, id := range ids { + id2 := *id + id2.ModTime = time.Time{} + + cleanIDs = append(cleanIDs, &id2) + } + + assert.ElementsMatch(t, expected, cleanIDs) }) } } @@ -627,7 +657,7 @@ func (suite *ModelStoreIntegrationSuite) TestPutUpdate() { err = m.GetWithModelStoreID(ctx, theModelType, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) ids, err := m.GetIDsForType(ctx, theModelType, nil) require.NoError(t, err, clues.ToCore(err)) @@ -822,7 +852,7 @@ func (suite *ModelStoreRegressionSuite) TestFailDuringWriteSessionHasNoVisibleEf err = m.GetWithModelStoreID(ctx, theModelType, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) } func openConnAndModelStore( diff --git a/src/internal/model/model.go b/src/internal/model/model.go index a3f25c820..fb72e3613 100644 --- a/src/internal/model/model.go +++ b/src/internal/model/model.go @@ -1,6 +1,8 @@ package model import ( + "time" + "github.com/kopia/kopia/repo/manifest" ) @@ -68,7 +70,8 @@ type BaseModel struct { // Tags associated with this model in the store to facilitate lookup. Tags in // the struct are not serialized directly into the stored model, but are part // of the metadata for the model. - Tags map[string]string `json:"-"` + Tags map[string]string `json:"-"` + ModTime time.Time `json:"-"` } func (bm *BaseModel) Base() *BaseModel {