diff --git a/src/internal/common/str/str.go b/src/internal/common/str/str.go index ac9caf7d3..41919b2b7 100644 --- a/src/internal/common/str/str.go +++ b/src/internal/common/str/str.go @@ -1,7 +1,9 @@ package str import ( + "encoding/hex" "fmt" + "hash/crc32" "strconv" "github.com/alcionai/clues" @@ -90,3 +92,12 @@ func SliceToMap(ss []string) map[string]struct{} { return m } + +func GenerateHash(input []byte) string { + crc32Hash := crc32.NewIEEE() + crc32Hash.Write(input) + checksum := crc32Hash.Sum(nil) + hashString := hex.EncodeToString(checksum) + + return hashString +} diff --git a/src/internal/common/str/str_test.go b/src/internal/common/str/str_test.go index 46a841966..11af84e93 100644 --- a/src/internal/common/str/str_test.go +++ b/src/internal/common/str/str_test.go @@ -1,9 +1,11 @@ package str import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // --------------------------------------------------------------------------- @@ -51,3 +53,68 @@ func TestPreview(t *testing.T) { }) } } + +// Test GenerateHash +func TestGenerateHash(t *testing.T) { + type testStruct struct { + Text string + Number int + Status bool + } + + table := []struct { + name string + input1 any + input2 any + sameCheck bool + }{ + { + name: "check if same hash is generated for same string input", + input1: "test data", + sameCheck: true, + }, + { + name: "check if same hash is generated for same struct input", + input1: testStruct{Text: "test text", Number: 1, Status: true}, + sameCheck: true, + }, + { + name: "check if different hash is generated for different string input", + input1: "test data", + input2: "test data 2", + sameCheck: false, + }, + { + name: "check if different hash is generated for different struct input", + input1: testStruct{Text: "test text", Number: 1, Status: true}, + input2: testStruct{Text: "test text 2", Number: 2, Status: false}, + sameCheck: false, + }, + } + + for _, test := range table { + var input1Bytes []byte + + var err error + + var hash1 string + + input1Bytes, err = json.Marshal(test.input1) + require.NoError(t, err) + + hash1 = GenerateHash(input1Bytes) + + if test.sameCheck { + hash2 := GenerateHash(input1Bytes) + + assert.Equal(t, hash1, hash2) + } else { + input2Bytes, err := json.Marshal(test.input2) + require.NoError(t, err) + + hash2 := GenerateHash(input2Bytes) + + assert.NotEqual(t, hash1, hash2) + } + } +} diff --git a/src/internal/common/str/testdata/str.go b/src/internal/common/str/testdata/str.go new file mode 100644 index 000000000..f95806945 --- /dev/null +++ b/src/internal/common/str/testdata/str.go @@ -0,0 +1,9 @@ +package testdata + +import "github.com/google/uuid" + +const hashLength = 7 + +func NewHashForRepoConfigName() string { + return uuid.NewString()[:hashLength] +} diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index 7a4948787..453993ec1 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -2,6 +2,7 @@ package kopia import ( "context" + "fmt" "path/filepath" "sync" "time" @@ -27,9 +28,9 @@ import ( ) const ( - defaultKopiaConfigDir = "/tmp/" - defaultKopiaConfigFile = "repository.config" - defaultCompressor = "zstd-better-compression" + defaultKopiaConfigDir = "/tmp/" + kopiaConfigFileTemplate = "repository-%s.config" + defaultCompressor = "zstd-better-compression" // Interval of 0 disables scheduling. defaultSchedulingInterval = time.Second * 0 ) @@ -95,6 +96,7 @@ func (w *conn) Initialize( ctx context.Context, opts repository.Options, retentionOpts repository.Retention, + repoNameHash string, ) error { bst, err := blobStoreByProvider(ctx, opts, w.storage) if err != nil { @@ -135,6 +137,7 @@ func (w *conn) Initialize( ctx, opts, cfg.KopiaCfgDir, + repoNameHash, bst, cfg.CorsoPassphrase, defaultCompressor) @@ -152,7 +155,7 @@ func (w *conn) Initialize( return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil() } -func (w *conn) Connect(ctx context.Context, opts repository.Options) error { +func (w *conn) Connect(ctx context.Context, opts repository.Options, repoNameHash string) error { bst, err := blobStoreByProvider(ctx, opts, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") @@ -168,6 +171,7 @@ func (w *conn) Connect(ctx context.Context, opts repository.Options) error { ctx, opts, cfg.KopiaCfgDir, + repoNameHash, bst, cfg.CorsoPassphrase, defaultCompressor) @@ -177,6 +181,7 @@ func (w *conn) commonConnect( ctx context.Context, opts repository.Options, configDir string, + repoNameHash string, bst blob.Storage, password, compressor string, ) error { @@ -196,7 +201,7 @@ func (w *conn) commonConnect( configDir = defaultKopiaConfigDir } - cfgFile := filepath.Join(configDir, defaultKopiaConfigFile) + cfgFile := filepath.Join(configDir, fmt.Sprintf(kopiaConfigFileTemplate, repoNameHash)) // todo - issue #75: nil here should be storage.ConnectOptions() if err := repo.Connect( @@ -579,13 +584,18 @@ func (w *conn) SnapshotRoot(man *snapshot.Manifest) (fs.Entry, error) { return snapshotfs.SnapshotRoot(w.Repository, man) } -func (w *conn) UpdatePassword(ctx context.Context, password string, opts repository.Options) error { +func (w *conn) UpdatePassword( + ctx context.Context, + password string, + opts repository.Options, + repoNameHash string, +) error { if len(password) <= 0 { return clues.New("empty password provided") } kopiaRef := NewConn(w.storage) - if err := kopiaRef.Connect(ctx, opts); err != nil { + if err := kopiaRef.Connect(ctx, opts, repoNameHash); err != nil { return clues.Wrap(err, "connecting kopia client") } diff --git a/src/internal/kopia/conn_test.go b/src/internal/kopia/conn_test.go index e5c2dbdec..bb629a31f 100644 --- a/src/internal/kopia/conn_test.go +++ b/src/internal/kopia/conn_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -27,9 +28,10 @@ func openLocalKopiaRepo( ctx context.Context, //revive:disable-line:context-as-argument ) (*conn, error) { st := storeTD.NewFilesystemStorage(t) + repoNameHash := strTD.NewHashForRepoConfigName() k := NewConn(st) - if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { + if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash); err != nil { return nil, err } @@ -41,9 +43,10 @@ func openKopiaRepo( ctx context.Context, //revive:disable-line:context-as-argument ) (*conn, error) { st := storeTD.NewPrefixedS3Storage(t) + repoNameHash := strTD.NewHashForRepoConfigName() k := NewConn(st) - if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { + if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash); err != nil { return nil, err } @@ -91,6 +94,7 @@ func TestWrapperIntegrationSuite(t *testing.T) { func (suite *WrapperIntegrationSuite) TestRepoExistsError() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -98,19 +102,20 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() { st := storeTD.NewFilesystemStorage(t) k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err = k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) assert.Error(t, err, clues.ToCore(err)) assert.ErrorIs(t, err, ErrorRepoAlreadyExists) } func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -119,12 +124,13 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { st.Provider = storage.ProviderUnknown k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) assert.Error(t, err, clues.ToCore(err)) } func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -132,7 +138,7 @@ func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() { st := storeTD.NewFilesystemStorage(t) k := NewConn(st) - err := k.Connect(ctx, repository.Options{}) + err := k.Connect(ctx, repository.Options{}, repoNameHash) assert.Error(t, err, clues.ToCore(err)) } @@ -282,6 +288,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect newRetentionDaily := policy.OptionalInt(42) newRetention := policy.RetentionPolicy{KeepDaily: &newRetentionDaily} newSchedInterval := time.Second * 42 + repoNameHash := strTD.NewHashForRepoConfigName() table := []struct { name string @@ -376,7 +383,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Connect(ctx, repository.Options{}) + err = k.Connect(ctx, repository.Options{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) defer func() { @@ -393,6 +400,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -404,7 +412,7 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() { require.NoError(t, err, clues.ToCore(err)) // Re-open with Connect. - err = k.Connect(ctx, repository.Options{}) + err = k.Connect(ctx, repository.Options{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) @@ -413,6 +421,7 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() { func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -425,7 +434,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { st := storeTD.NewFilesystemStorage(t) k := NewConn(st) - err := k.Initialize(ctx, opts, repository.Retention{}) + err := k.Initialize(ctx, opts, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) kopiaOpts := k.ClientOptions() @@ -439,7 +448,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { opts.User = "hello" opts.Host = "world" - err = k.Connect(ctx, opts) + err = k.Connect(ctx, opts, repoNameHash) require.NoError(t, err, clues.ToCore(err)) kopiaOpts = k.ClientOptions() @@ -453,7 +462,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { opts.User = "" opts.Host = "" - err = k.Connect(ctx, opts) + err = k.Connect(ctx, opts, repoNameHash) require.NoError(t, err, clues.ToCore(err)) kopiaOpts = k.ClientOptions() @@ -485,6 +494,7 @@ func TestConnRetentionIntegrationSuite(t *testing.T) { // from the default values that kopia uses. func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -492,7 +502,7 @@ func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { st1 := storeTD.NewPrefixedS3Storage(t) k1 := NewConn(st1) - err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err)) st2 := storeTD.NewPrefixedS3Storage(t) @@ -505,7 +515,8 @@ func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { Mode: ptr.To(repository.GovernanceRetention), Duration: ptr.To(time.Hour * 48), Extend: ptr.To(true), - }) + }, + repoNameHash) require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err)) dr1, ok := k1.Repository.(repo.DirectRepository) diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index db25eee57..d346c60b4 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/tester" @@ -858,8 +859,9 @@ func openConnAndModelStore( ) (*conn, *ModelStore) { st := storeTD.NewFilesystemStorage(t) c := NewConn(st) + repoNameHash := strTD.NewHashForRepoConfigName() - err := c.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := c.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) defer func() { @@ -878,7 +880,8 @@ func reconnectToModelStore( ctx context.Context, //revive:disable-line:context-as-argument c *conn, ) *ModelStore { - err := c.Connect(ctx, repository.Options{}) + repoNameHash := strTD.NewHashForRepoConfigName() + err := c.Connect(ctx, repository.Options{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 9d365c92f..fc8c77c21 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -23,6 +23,7 @@ import ( pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" "github.com/alcionai/corso/src/internal/common/ptr" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" @@ -202,6 +203,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_FirstRun_NoChanges() { func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -228,7 +230,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails Host: "bar", } - err = k.Connect(ctx, opts) + err = k.Connect(ctx, opts, repoNameHash) require.NoError(t, err, clues.ToCore(err)) var notOwnedErr maintenance.NotOwnedError @@ -239,6 +241,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeeds() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -265,7 +268,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed Host: "bar", } - err = k.Connect(ctx, opts) + err = k.Connect(ctx, opts, repoNameHash) require.NoError(t, err, clues.ToCore(err)) mOpts.Force = true @@ -286,6 +289,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed // blobs as there's several of them, but at least this gives us something. func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnFailure() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -318,7 +322,7 @@ func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnF k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Connect(ctx, repository.Options{}) + err = k.Connect(ctx, repository.Options{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) defer k.Close(ctx) @@ -375,6 +379,7 @@ func checkRetentionParams( //revive:disable-next-line:context-as-argument func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) { k := w.c + repoNameHash := strTD.NewHashForRepoConfigName() err := w.Close(ctx) require.NoError(t, err, "closing wrapper: %v", clues.ToCore(err)) @@ -382,7 +387,7 @@ func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) { err = k.Close(ctx) require.NoError(t, err, "closing conn: %v", clues.ToCore(err)) - err = k.Connect(ctx, repository.Options{}) + err = k.Connect(ctx, repository.Options{}, repoNameHash) require.NoError(t, err, "reconnecting conn: %v", clues.ToCore(err)) w.c = k diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index b6479a875..d55ddd132 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/prefixmatcher" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -1514,6 +1515,7 @@ func TestAssistBackupIntegrationSuite(t *testing.T) { func (suite *AssistBackupIntegrationSuite) SetupSuite() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -1525,7 +1527,7 @@ func (suite *AssistBackupIntegrationSuite) SetupSuite() { suite.acct = tconfig.NewM365Account(t) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { diff --git a/src/internal/operations/maintenance_test.go b/src/internal/operations/maintenance_test.go index 1efe6599e..81269d6dd 100644 --- a/src/internal/operations/maintenance_test.go +++ b/src/internal/operations/maintenance_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -39,8 +40,9 @@ func getKopiaHandles( ctx context.Context, //revive:disable-line:context-as-argument ) (*kopia.Wrapper, *kopia.ModelStore) { st := storeTD.NewPrefixedS3Storage(t) + repoNameHash := strTD.NewHashForRepoConfigName() k := kopia.NewConn(st) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) kw, err := kopia.NewWrapper(k) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 7e0114678..2e92f8369 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/idname" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -236,13 +237,14 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { graph.InitializeConcurrencyLimiter(ctx, true, 4) var ( - st = storeTD.NewPrefixedS3Storage(t) - k = kopia.NewConn(st) + st = storeTD.NewPrefixedS3Storage(t) + k = kopia.NewConn(st) + repoNameHash = strTD.NewHashForRepoConfigName() ) suite.acct = tconfig.NewM365Account(t) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { diff --git a/src/internal/operations/retention_config_test.go b/src/internal/operations/retention_config_test.go index ce57cd879..5b4458643 100644 --- a/src/internal/operations/retention_config_test.go +++ b/src/internal/operations/retention_config_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/tester" @@ -33,14 +34,15 @@ func (suite *RetentionConfigOpIntegrationSuite) TestRepoRetentionConfig() { var ( t = suite.T() // need to initialize the repository before we can test connecting to it. - st = storeTD.NewPrefixedS3Storage(t) - k = kopia.NewConn(st) + st = storeTD.NewPrefixedS3Storage(t) + k = kopia.NewConn(st) + repoNameHash = strTD.NewHashForRepoConfigName() ) ctx, flush := tester.NewContext(t) defer flush() - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) kw, err := kopia.NewWrapper(k) diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index b5079556c..1dd34e01b 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -124,10 +125,11 @@ func prepNewTestBackupOp( acct: tconfig.NewM365Account(t), st: storeTD.NewPrefixedS3Storage(t), } + repoNameHash := strTD.NewHashForRepoConfigName() k := kopia.NewConn(bod.st) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/operations/test/restore_helper_test.go b/src/internal/operations/test/restore_helper_test.go index 36356fdd2..63dd99649 100644 --- a/src/internal/operations/test/restore_helper_test.go +++ b/src/internal/operations/test/restore_helper_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/alcionai/corso/src/internal/common/idname" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" @@ -81,10 +82,11 @@ func prepNewTestRestoreOp( acct: tconfig.NewM365Account(t), st: backupStore, } - k = kopia.NewConn(rod.st) + k = kopia.NewConn(rod.st) + repoNameHash = strTD.NewHashForRepoConfigName() ) - err := k.Connect(ctx, repository.Options{}) + err := k.Connect(ctx, repository.Options{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) // kopiaRef comes with a count of 1 and Wrapper bumps it again diff --git a/src/internal/streamstore/collectables_test.go b/src/internal/streamstore/collectables_test.go index 5257ee718..848e0642f 100644 --- a/src/internal/streamstore/collectables_test.go +++ b/src/internal/streamstore/collectables_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/tester" @@ -36,6 +37,7 @@ func TestStreamStoreIntgSuite(t *testing.T) { func (suite *StreamStoreIntgSuite) SetupSubTest() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -44,7 +46,7 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() { st := storeTD.NewPrefixedS3Storage(t) k := kopia.NewConn(st) - err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) suite.kcloser = func() { k.Close(ctx) } diff --git a/src/pkg/account/account.go b/src/pkg/account/account.go index ea1eb3070..f4b5e82ba 100644 --- a/src/pkg/account/account.go +++ b/src/pkg/account/account.go @@ -17,6 +17,7 @@ const ( // storage parsing errors var ( errMissingRequired = clues.New("missing required storage configuration") + errInvalidProvider = clues.New("unsupported account provider") ) const ( @@ -38,6 +39,7 @@ type providerIDer interface { common.StringConfigurer providerID(accountProvider) string + configHash() (string, error) } // NewAccount aggregates all the supplied configurations into a single configuration @@ -88,3 +90,17 @@ func (a Account) ID() string { return a.Config[a.Provider.String()+"-tenant-id"] } + +func (a Account) GetAccountConfigHash() (string, error) { + switch a.Provider { + case ProviderM365: + m365, err := a.M365Config() + if err != nil { + return "", clues.Stack(err) + } + + return m365.configHash() + } + + return "", errInvalidProvider.With("provider", a.Provider) +} diff --git a/src/pkg/account/account_test.go b/src/pkg/account/account_test.go index 01279666a..cf00e2a8e 100644 --- a/src/pkg/account/account_test.go +++ b/src/pkg/account/account_test.go @@ -5,6 +5,7 @@ import ( "github.com/alcionai/clues" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -22,6 +23,10 @@ func (c testConfig) StringConfig() (map[string]string, error) { return map[string]string{"expect": c.expect}, c.err } +func (c testConfig) configHash() (string, error) { + return "hashed-config", c.err +} + type AccountSuite struct { suite.Suite } @@ -63,3 +68,57 @@ func (suite *AccountSuite) TestNewAccount() { }) } } + +func (suite *AccountSuite) TestGetAccountConfigHash() { + tests := []struct { + name string + provider accountProvider + config any + }{ + { + name: "valid account", + provider: ProviderM365, + config: getTestM365Config("1234", "5678"), + }, + { + name: "invalid account", + provider: ProviderUnknown, + config: testConfig{"configVal", "", nil}, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + if test.provider == ProviderUnknown { + s, err := NewAccount(test.provider, test.config.(testConfig)) + require.NoError(t, err) + + _, err = s.GetAccountConfigHash() + require.Error(t, err) + } + + if test.provider == ProviderM365 { + _, ok := test.config.(providerIDer) + require.True(t, ok) + + s, err := NewAccount(test.provider, test.config.(M365Config)) + require.NoError(t, err) + + hash, err := s.GetAccountConfigHash() + require.NoError(t, err) + assert.True(t, len(hash) > 0) + } + }) + } +} + +func getTestM365Config(clientID, tenantID string) M365Config { + c := M365Config{} + c.AzureClientID = clientID + c.AzureClientSecret = "super secret" + c.AzureTenantID = tenantID + + return c +} diff --git a/src/pkg/account/m365.go b/src/pkg/account/m365.go index 38d6efc88..4d8c3302c 100644 --- a/src/pkg/account/m365.go +++ b/src/pkg/account/m365.go @@ -1,8 +1,13 @@ package account import ( + "encoding/json" + "reflect" + "slices" + "github.com/alcionai/clues" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/credentials" ) @@ -11,6 +16,8 @@ const ( AzureTenantID = "AZURE_TENANT_ID" ) +var excludedM365ConfigFieldsForHashing = []string{"AzureClientSecret"} + type M365Config struct { credentials.M365 // requires: ClientID, ClientSecret AzureTenantID string @@ -45,6 +52,17 @@ func (c M365Config) providerID(ap accountProvider) string { return "" } +func (c M365Config) configHash() (string, error) { + filteredM365Config := createFilteredM365ConfigForHashing(c) + + b, err := json.Marshal(filteredM365Config) + if err != nil { + return "", clues.Stack(err) + } + + return str.GenerateHash(b), nil +} + // M365Config retrieves the M365Config details from the Account config. func (a Account) M365Config() (M365Config, error) { c := M365Config{} @@ -57,6 +75,20 @@ func (a Account) M365Config() (M365Config, error) { return c, c.validate() } +func createFilteredM365ConfigForHashing(source M365Config) map[string]any { + filteredM365Config := make(map[string]any) + sourceValue := reflect.ValueOf(source) + + for i := 0; i < sourceValue.NumField(); i++ { + fieldName := sourceValue.Type().Field(i).Name + if !slices.Contains(excludedM365ConfigFieldsForHashing, fieldName) { + filteredM365Config[fieldName] = sourceValue.Field(i).Interface() + } + } + + return filteredM365Config +} + func (c M365Config) validate() error { check := map[string]string{ credentials.AzureClientID: c.AzureClientID, diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index db1ecd510..b193c63fa 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -2,6 +2,7 @@ package repository import ( "context" + "fmt" "time" "github.com/alcionai/clues" @@ -211,12 +212,17 @@ func (r *repository) UpdatePassword(ctx context.Context, password string) (err e progressBar := observe.MessageWithCompletion(ctx, "Connecting to repository") defer close(progressBar) + repoNameHash, err := r.GenerateHashForRepositoryConfigFileName() + if err != nil { + return clues.Wrap(err, "generating repo config hash") + } + kopiaRef := kopia.NewConn(r.Storage) - if err := kopiaRef.Connect(ctx, r.Opts.Repo); err != nil { + if err := kopiaRef.Connect(ctx, r.Opts.Repo, repoNameHash); err != nil { return clues.Wrap(err, "connecting kopia client") } - err = kopiaRef.UpdatePassword(ctx, password, r.Opts.Repo) + err = kopiaRef.UpdatePassword(ctx, password, r.Opts.Repo, repoNameHash) if err != nil { return clues.Wrap(err, "updating on kopia") } @@ -294,9 +300,14 @@ func (r *repository) setupKopia( ) error { var err error + repoHashName, err := r.GenerateHashForRepositoryConfigFileName() + if err != nil { + return clues.Wrap(err, "generating repo config hash") + } + kopiaRef := kopia.NewConn(r.Storage) if isInitialize { - if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts); err != nil { + if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts, repoHashName); err != nil { // Replace common internal errors so that SDK users can check results with errors.Is() if errors.Is(err, kopia.ErrorRepoAlreadyExists) { return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) @@ -305,7 +316,7 @@ func (r *repository) setupKopia( return clues.Wrap(err, "initializing kopia") } } else { - if err := kopiaRef.Connect(ctx, r.Opts.Repo); err != nil { + if err := kopiaRef.Connect(ctx, r.Opts.Repo, repoHashName); err != nil { return clues.Wrap(err, "connecting kopia client") } } @@ -335,6 +346,20 @@ func (r *repository) setupKopia( return nil } +func (r repository) GenerateHashForRepositoryConfigFileName() (string, error) { + accountHash, err := r.Account.GetAccountConfigHash() + if err != nil { + return "", clues.Wrap(err, "fetch account config hash") + } + + storageHash, err := r.Storage.GetStorageConfigHash() + if err != nil { + return "", clues.Wrap(err, "fetch storage config hash") + } + + return fmt.Sprintf("%s-%s", accountHash, storageHash), nil +} + // --------------------------------------------------------------------------- // Repository ID Model // --------------------------------------------------------------------------- diff --git a/src/pkg/repository/repository_unexported_test.go b/src/pkg/repository/repository_unexported_test.go index ea15899e8..d57db9cf5 100644 --- a/src/pkg/repository/repository_unexported_test.go +++ b/src/pkg/repository/repository_unexported_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + strTD "github.com/alcionai/corso/src/internal/common/str/testdata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/model" @@ -698,6 +699,7 @@ func TestRepositoryModelIntgSuite(t *testing.T) { func (suite *RepositoryModelIntgSuite) SetupSuite() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -710,10 +712,10 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() { require.NotNil(t, k) - err = k.Initialize(ctx, rep.Options{}, rep.Retention{}) + err = k.Initialize(ctx, rep.Options{}, rep.Retention{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) - err = k.Connect(ctx, rep.Options{}) + err = k.Connect(ctx, rep.Options{}, repoNameHash) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { @@ -752,6 +754,7 @@ func (suite *RepositoryModelIntgSuite) TearDownSuite() { func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { t := suite.T() + repoNameHash := strTD.NewHashForRepoConfigName() ctx, flush := tester.NewContext(t) defer flush() @@ -761,10 +764,10 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { k = kopia.NewConn(s) ) - err := k.Initialize(ctx, rep.Options{}, rep.Retention{}) + err := k.Initialize(ctx, rep.Options{}, rep.Retention{}, repoNameHash) require.NoError(t, err, "initializing repo: %v", clues.ToCore(err)) - err = k.Connect(ctx, rep.Options{}) + err = k.Connect(ctx, rep.Options{}, repoNameHash) require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err)) defer k.Close(ctx) diff --git a/src/pkg/storage/filesystem.go b/src/pkg/storage/filesystem.go index 2826ad211..e4293283f 100644 --- a/src/pkg/storage/filesystem.go +++ b/src/pkg/storage/filesystem.go @@ -1,6 +1,9 @@ package storage import ( + "encoding/json" + "reflect" + "slices" "strings" "github.com/alcionai/clues" @@ -10,6 +13,9 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +// nothing to exclude, for parity +var excludedFileSystemConfigFieldsForHashing = []string{} + const ( FilesystemPath = "path" ) @@ -58,6 +64,31 @@ func (c *FilesystemConfig) fsConfigsFromStore(g Getter) { c.Path = cast.ToString(g.Get(FilesystemPath)) } +func (c FilesystemConfig) configHash() (string, error) { + filteredFileSystemConfig := createFilteredFileSystemConfigForHashing(c) + + b, err := json.Marshal(filteredFileSystemConfig) + if err != nil { + return "", clues.Stack(err) + } + + return str.GenerateHash(b), nil +} + +func createFilteredFileSystemConfigForHashing(source FilesystemConfig) map[string]any { + filteredFileSystemConfig := make(map[string]any) + sourceValue := reflect.ValueOf(source) + + for i := 0; i < sourceValue.NumField(); i++ { + fieldName := sourceValue.Type().Field(i).Name + if !slices.Contains(excludedFileSystemConfigFieldsForHashing, fieldName) { + filteredFileSystemConfig[fieldName] = sourceValue.Field(i).Interface() + } + } + + return filteredFileSystemConfig +} + // TODO(pandeyabs): Remove this. It's not adding any value. func fsOverrides(in map[string]string) map[string]string { return map[string]string{ diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index 825228c23..5de425042 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -1,7 +1,10 @@ package storage import ( + "encoding/json" "os" + "reflect" + "slices" "strconv" "github.com/alcionai/clues" @@ -21,6 +24,14 @@ type S3Config struct { DoNotVerifyTLS bool } +var excludedS3ConfigFieldsForHashing = []string{ + "DoNotUseTLS", + "DoNotVerifyTLS", + "AccessKey", + "SecretKey", + "SessionToken", +} + // config key consts const ( keyS3AccessKey = "s3_access_key" @@ -129,6 +140,31 @@ func (c S3Config) validate() error { return nil } +func (c S3Config) configHash() (string, error) { + filteredS3Config := createFilteredS3ConfigForHashing(c) + + b, err := json.Marshal(filteredS3Config) + if err != nil { + return "", clues.Stack(err) + } + + return str.GenerateHash(b), nil +} + +func createFilteredS3ConfigForHashing(source S3Config) map[string]any { + filteredS3Config := make(map[string]any) + sourceValue := reflect.ValueOf(source) + + for i := 0; i < sourceValue.NumField(); i++ { + fieldName := sourceValue.Type().Field(i).Name + if !slices.Contains(excludedS3ConfigFieldsForHashing, fieldName) { + filteredS3Config[fieldName] = sourceValue.Field(i).Interface() + } + } + + return filteredS3Config +} + func s3Overrides(in map[string]string) map[string]string { return map[string]string{ Bucket: in[Bucket], diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index cb7ee4791..89a713ecb 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -35,6 +35,7 @@ const ( // storage parsing errors var ( errMissingRequired = clues.New("missing required storage configuration") + errInvalidProvider = clues.New("unsupported storage provider") ) // Storage defines a storage provider, along with any configuration @@ -106,7 +107,29 @@ func (s Storage) StorageConfig() (Configurer, error) { return buildFilesystemConfigFromMap(s.Config) } - return nil, clues.New("unsupported storage provider: [" + s.Provider.String() + "]") + return nil, errInvalidProvider.With("provider", s.Provider) +} + +func (s Storage) GetStorageConfigHash() (string, error) { + switch s.Provider { + case ProviderS3: + s3Cnf, err := s.ToS3Config() + if err != nil { + return "", err + } + + return s3Cnf.configHash() + + case ProviderFilesystem: + fsCnf, err := s.ToFilesystemConfig() + if err != nil { + return "", err + } + + return fsCnf.configHash() + } + + return "", errInvalidProvider.With("provider", s.Provider) } func NewStorageConfig(provider ProviderType) (Configurer, error) { @@ -117,7 +140,7 @@ func NewStorageConfig(provider ProviderType) (Configurer, error) { return &FilesystemConfig{}, nil } - return nil, clues.New("unsupported storage provider: [" + provider.String() + "]") + return nil, errInvalidProvider.With("provider", provider) } type Getter interface { @@ -149,6 +172,8 @@ type Configurer interface { ) error WriteConfigToStorer + + configHash() (string, error) } // mustMatchConfig compares the values of each key to their config file value in store. diff --git a/src/pkg/storage/storage_test.go b/src/pkg/storage/storage_test.go index ccba54bc6..6112a6339 100644 --- a/src/pkg/storage/storage_test.go +++ b/src/pkg/storage/storage_test.go @@ -5,6 +5,7 @@ import ( "github.com/alcionai/clues" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" @@ -63,6 +64,84 @@ func (suite *StorageUnitSuite) TestNewStorage() { } } +func (suite *StorageUnitSuite) TestGetAccountConfigHash() { + tests := []struct { + name string + provider ProviderType + config any + }{ + { + name: "s3 storage", + provider: ProviderS3, + config: getTestS3Config("test-bucket", "https://aws.s3", "test-prefix"), + }, + { + name: "filesystem storage", + provider: ProviderFilesystem, + config: getTestFileSystemConfig("test/to/dir"), + }, + { + name: "invalid account", + provider: ProviderUnknown, + config: testConfig{"configVal", nil}, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + if test.provider == ProviderUnknown { + s, err := NewStorage(test.provider, test.config.(testConfig)) + require.NoError(t, err) + + _, err = s.GetStorageConfigHash() + require.Error(t, err) + } + + if test.provider == ProviderS3 { + _, ok := test.config.(Configurer) + require.True(t, ok) + + s3Cnf := test.config.(*S3Config) + s, err := NewStorage(test.provider, s3Cnf) + require.NoError(t, err) + + hash, err := s.GetStorageConfigHash() + require.NoError(t, err) + assert.True(t, len(hash) > 0) + } + + if test.provider == ProviderFilesystem { + _, ok := test.config.(Configurer) + require.True(t, ok) + + fsCnf := test.config.(*FilesystemConfig) + s, err := NewStorage(test.provider, fsCnf) + require.NoError(t, err) + + hash, err := s.GetStorageConfigHash() + require.NoError(t, err) + assert.True(t, len(hash) > 0) + } + }) + } +} + +func getTestS3Config(bucket, endpoint, prefix string) *S3Config { + return &S3Config{ + Bucket: bucket, + Endpoint: endpoint, + Prefix: prefix, + } +} + +func getTestFileSystemConfig(path string) *FilesystemConfig { + return &FilesystemConfig{ + Path: path, + } +} + type testGetter struct { storeMap map[string]string }