generate repository config name based on provider specific hash (#4639)

Enable user/client to be able to perform multiple backups in a concurrent manner irrespective of storage provider (s3/filesystem) for different tenants (anything that differentiates one account from another).

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ]  No

#### Type of change

- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #4443 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Hitesh Pattanayak 2023-11-11 14:57:06 +05:30 committed by GitHub
parent 0b997e6176
commit 40c7476e24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 484 additions and 48 deletions

View File

@ -1,7 +1,9 @@
package str package str
import ( import (
"encoding/hex"
"fmt" "fmt"
"hash/crc32"
"strconv" "strconv"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -90,3 +92,12 @@ func SliceToMap(ss []string) map[string]struct{} {
return m return m
} }
func GenerateHash(input []byte) string {
crc32Hash := crc32.NewIEEE()
crc32Hash.Write(input)
checksum := crc32Hash.Sum(nil)
hashString := hex.EncodeToString(checksum)
return hashString
}

View File

@ -1,9 +1,11 @@
package str package str
import ( import (
"encoding/json"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -51,3 +53,68 @@ func TestPreview(t *testing.T) {
}) })
} }
} }
// Test GenerateHash
func TestGenerateHash(t *testing.T) {
type testStruct struct {
Text string
Number int
Status bool
}
table := []struct {
name string
input1 any
input2 any
sameCheck bool
}{
{
name: "check if same hash is generated for same string input",
input1: "test data",
sameCheck: true,
},
{
name: "check if same hash is generated for same struct input",
input1: testStruct{Text: "test text", Number: 1, Status: true},
sameCheck: true,
},
{
name: "check if different hash is generated for different string input",
input1: "test data",
input2: "test data 2",
sameCheck: false,
},
{
name: "check if different hash is generated for different struct input",
input1: testStruct{Text: "test text", Number: 1, Status: true},
input2: testStruct{Text: "test text 2", Number: 2, Status: false},
sameCheck: false,
},
}
for _, test := range table {
var input1Bytes []byte
var err error
var hash1 string
input1Bytes, err = json.Marshal(test.input1)
require.NoError(t, err)
hash1 = GenerateHash(input1Bytes)
if test.sameCheck {
hash2 := GenerateHash(input1Bytes)
assert.Equal(t, hash1, hash2)
} else {
input2Bytes, err := json.Marshal(test.input2)
require.NoError(t, err)
hash2 := GenerateHash(input2Bytes)
assert.NotEqual(t, hash1, hash2)
}
}
}

View File

@ -0,0 +1,9 @@
package testdata
import "github.com/google/uuid"
const hashLength = 7
func NewHashForRepoConfigName() string {
return uuid.NewString()[:hashLength]
}

View File

@ -2,6 +2,7 @@ package kopia
import ( import (
"context" "context"
"fmt"
"path/filepath" "path/filepath"
"sync" "sync"
"time" "time"
@ -28,7 +29,7 @@ import (
const ( const (
defaultKopiaConfigDir = "/tmp/" defaultKopiaConfigDir = "/tmp/"
defaultKopiaConfigFile = "repository.config" kopiaConfigFileTemplate = "repository-%s.config"
defaultCompressor = "zstd-better-compression" defaultCompressor = "zstd-better-compression"
// Interval of 0 disables scheduling. // Interval of 0 disables scheduling.
defaultSchedulingInterval = time.Second * 0 defaultSchedulingInterval = time.Second * 0
@ -95,6 +96,7 @@ func (w *conn) Initialize(
ctx context.Context, ctx context.Context,
opts repository.Options, opts repository.Options,
retentionOpts repository.Retention, retentionOpts repository.Retention,
repoNameHash string,
) error { ) error {
bst, err := blobStoreByProvider(ctx, opts, w.storage) bst, err := blobStoreByProvider(ctx, opts, w.storage)
if err != nil { if err != nil {
@ -135,6 +137,7 @@ func (w *conn) Initialize(
ctx, ctx,
opts, opts,
cfg.KopiaCfgDir, cfg.KopiaCfgDir,
repoNameHash,
bst, bst,
cfg.CorsoPassphrase, cfg.CorsoPassphrase,
defaultCompressor) defaultCompressor)
@ -152,7 +155,7 @@ func (w *conn) Initialize(
return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil() return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil()
} }
func (w *conn) Connect(ctx context.Context, opts repository.Options) error { func (w *conn) Connect(ctx context.Context, opts repository.Options, repoNameHash string) error {
bst, err := blobStoreByProvider(ctx, opts, w.storage) bst, err := blobStoreByProvider(ctx, opts, w.storage)
if err != nil { if err != nil {
return clues.Wrap(err, "initializing storage") return clues.Wrap(err, "initializing storage")
@ -168,6 +171,7 @@ func (w *conn) Connect(ctx context.Context, opts repository.Options) error {
ctx, ctx,
opts, opts,
cfg.KopiaCfgDir, cfg.KopiaCfgDir,
repoNameHash,
bst, bst,
cfg.CorsoPassphrase, cfg.CorsoPassphrase,
defaultCompressor) defaultCompressor)
@ -177,6 +181,7 @@ func (w *conn) commonConnect(
ctx context.Context, ctx context.Context,
opts repository.Options, opts repository.Options,
configDir string, configDir string,
repoNameHash string,
bst blob.Storage, bst blob.Storage,
password, compressor string, password, compressor string,
) error { ) error {
@ -196,7 +201,7 @@ func (w *conn) commonConnect(
configDir = defaultKopiaConfigDir configDir = defaultKopiaConfigDir
} }
cfgFile := filepath.Join(configDir, defaultKopiaConfigFile) cfgFile := filepath.Join(configDir, fmt.Sprintf(kopiaConfigFileTemplate, repoNameHash))
// todo - issue #75: nil here should be storage.ConnectOptions() // todo - issue #75: nil here should be storage.ConnectOptions()
if err := repo.Connect( if err := repo.Connect(
@ -579,13 +584,18 @@ func (w *conn) SnapshotRoot(man *snapshot.Manifest) (fs.Entry, error) {
return snapshotfs.SnapshotRoot(w.Repository, man) return snapshotfs.SnapshotRoot(w.Repository, man)
} }
func (w *conn) UpdatePassword(ctx context.Context, password string, opts repository.Options) error { func (w *conn) UpdatePassword(
ctx context.Context,
password string,
opts repository.Options,
repoNameHash string,
) error {
if len(password) <= 0 { if len(password) <= 0 {
return clues.New("empty password provided") return clues.New("empty password provided")
} }
kopiaRef := NewConn(w.storage) kopiaRef := NewConn(w.storage)
if err := kopiaRef.Connect(ctx, opts); err != nil { if err := kopiaRef.Connect(ctx, opts, repoNameHash); err != nil {
return clues.Wrap(err, "connecting kopia client") return clues.Wrap(err, "connecting kopia client")
} }

View File

@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage"
@ -27,9 +28,10 @@ func openLocalKopiaRepo(
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
) (*conn, error) { ) (*conn, error) {
st := storeTD.NewFilesystemStorage(t) st := storeTD.NewFilesystemStorage(t)
repoNameHash := strTD.NewHashForRepoConfigName()
k := NewConn(st) k := NewConn(st)
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash); err != nil {
return nil, err return nil, err
} }
@ -41,9 +43,10 @@ func openKopiaRepo(
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
) (*conn, error) { ) (*conn, error) {
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
repoNameHash := strTD.NewHashForRepoConfigName()
k := NewConn(st) k := NewConn(st)
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash); err != nil {
return nil, err return nil, err
} }
@ -91,6 +94,7 @@ func TestWrapperIntegrationSuite(t *testing.T) {
func (suite *WrapperIntegrationSuite) TestRepoExistsError() { func (suite *WrapperIntegrationSuite) TestRepoExistsError() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -98,19 +102,20 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() {
st := storeTD.NewFilesystemStorage(t) st := storeTD.NewFilesystemStorage(t)
k := NewConn(st) k := NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = k.Close(ctx) err = k.Close(ctx)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = k.Initialize(ctx, repository.Options{}, repository.Retention{}) err = k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
assert.Error(t, err, clues.ToCore(err)) assert.Error(t, err, clues.ToCore(err))
assert.ErrorIs(t, err, ErrorRepoAlreadyExists) assert.ErrorIs(t, err, ErrorRepoAlreadyExists)
} }
func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { func (suite *WrapperIntegrationSuite) TestBadProviderErrors() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -119,12 +124,13 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() {
st.Provider = storage.ProviderUnknown st.Provider = storage.ProviderUnknown
k := NewConn(st) k := NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
assert.Error(t, err, clues.ToCore(err)) assert.Error(t, err, clues.ToCore(err))
} }
func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() { func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -132,7 +138,7 @@ func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() {
st := storeTD.NewFilesystemStorage(t) st := storeTD.NewFilesystemStorage(t)
k := NewConn(st) k := NewConn(st)
err := k.Connect(ctx, repository.Options{}) err := k.Connect(ctx, repository.Options{}, repoNameHash)
assert.Error(t, err, clues.ToCore(err)) assert.Error(t, err, clues.ToCore(err))
} }
@ -282,6 +288,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect
newRetentionDaily := policy.OptionalInt(42) newRetentionDaily := policy.OptionalInt(42)
newRetention := policy.RetentionPolicy{KeepDaily: &newRetentionDaily} newRetention := policy.RetentionPolicy{KeepDaily: &newRetentionDaily}
newSchedInterval := time.Second * 42 newSchedInterval := time.Second * 42
repoNameHash := strTD.NewHashForRepoConfigName()
table := []struct { table := []struct {
name string name string
@ -376,7 +383,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect
err = k.Close(ctx) err = k.Close(ctx)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, repository.Options{}) err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
defer func() { defer func() {
@ -393,6 +400,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect
func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() { func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -404,7 +412,7 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() {
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// Re-open with Connect. // Re-open with Connect.
err = k.Connect(ctx, repository.Options{}) err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = k.Close(ctx) err = k.Close(ctx)
@ -413,6 +421,7 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() {
func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -425,7 +434,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
st := storeTD.NewFilesystemStorage(t) st := storeTD.NewFilesystemStorage(t)
k := NewConn(st) k := NewConn(st)
err := k.Initialize(ctx, opts, repository.Retention{}) err := k.Initialize(ctx, opts, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
kopiaOpts := k.ClientOptions() kopiaOpts := k.ClientOptions()
@ -439,7 +448,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
opts.User = "hello" opts.User = "hello"
opts.Host = "world" opts.Host = "world"
err = k.Connect(ctx, opts) err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
kopiaOpts = k.ClientOptions() kopiaOpts = k.ClientOptions()
@ -453,7 +462,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
opts.User = "" opts.User = ""
opts.Host = "" opts.Host = ""
err = k.Connect(ctx, opts) err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
kopiaOpts = k.ClientOptions() kopiaOpts = k.ClientOptions()
@ -485,6 +494,7 @@ func TestConnRetentionIntegrationSuite(t *testing.T) {
// from the default values that kopia uses. // from the default values that kopia uses.
func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -492,7 +502,7 @@ func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
st1 := storeTD.NewPrefixedS3Storage(t) st1 := storeTD.NewPrefixedS3Storage(t)
k1 := NewConn(st1) k1 := NewConn(st1)
err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err)) require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err))
st2 := storeTD.NewPrefixedS3Storage(t) st2 := storeTD.NewPrefixedS3Storage(t)
@ -505,7 +515,8 @@ func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
Mode: ptr.To(repository.GovernanceRetention), Mode: ptr.To(repository.GovernanceRetention),
Duration: ptr.To(time.Hour * 48), Duration: ptr.To(time.Hour * 48),
Extend: ptr.To(true), Extend: ptr.To(true),
}) },
repoNameHash)
require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err)) require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err))
dr1, ok := k1.Repository.(repo.DirectRepository) dr1, ok := k1.Repository.(repo.DirectRepository)

View File

@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -858,8 +859,9 @@ func openConnAndModelStore(
) (*conn, *ModelStore) { ) (*conn, *ModelStore) {
st := storeTD.NewFilesystemStorage(t) st := storeTD.NewFilesystemStorage(t)
c := NewConn(st) c := NewConn(st)
repoNameHash := strTD.NewHashForRepoConfigName()
err := c.Initialize(ctx, repository.Options{}, repository.Retention{}) err := c.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
defer func() { defer func() {
@ -878,7 +880,8 @@ func reconnectToModelStore(
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
c *conn, c *conn,
) *ModelStore { ) *ModelStore {
err := c.Connect(ctx, repository.Options{}) repoNameHash := strTD.NewHashForRepoConfigName()
err := c.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
defer func() { defer func() {

View File

@ -23,6 +23,7 @@ import (
pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
@ -202,6 +203,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_FirstRun_NoChanges() {
func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails() { func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -228,7 +230,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails
Host: "bar", Host: "bar",
} }
err = k.Connect(ctx, opts) err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
var notOwnedErr maintenance.NotOwnedError var notOwnedErr maintenance.NotOwnedError
@ -239,6 +241,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails
func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeeds() { func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeeds() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -265,7 +268,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed
Host: "bar", Host: "bar",
} }
err = k.Connect(ctx, opts) err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
mOpts.Force = true mOpts.Force = true
@ -286,6 +289,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed
// blobs as there's several of them, but at least this gives us something. // blobs as there's several of them, but at least this gives us something.
func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnFailure() { func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnFailure() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -318,7 +322,7 @@ func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnF
k.Close(ctx) k.Close(ctx)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, repository.Options{}) err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
defer k.Close(ctx) defer k.Close(ctx)
@ -375,6 +379,7 @@ func checkRetentionParams(
//revive:disable-next-line:context-as-argument //revive:disable-next-line:context-as-argument
func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) { func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) {
k := w.c k := w.c
repoNameHash := strTD.NewHashForRepoConfigName()
err := w.Close(ctx) err := w.Close(ctx)
require.NoError(t, err, "closing wrapper: %v", clues.ToCore(err)) require.NoError(t, err, "closing wrapper: %v", clues.ToCore(err))
@ -382,7 +387,7 @@ func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) {
err = k.Close(ctx) err = k.Close(ctx)
require.NoError(t, err, "closing conn: %v", clues.ToCore(err)) require.NoError(t, err, "closing conn: %v", clues.ToCore(err))
err = k.Connect(ctx, repository.Options{}) err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, "reconnecting conn: %v", clues.ToCore(err)) require.NoError(t, err, "reconnecting conn: %v", clues.ToCore(err))
w.c = k w.c = k

View File

@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/prefixmatcher"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -1514,6 +1515,7 @@ func TestAssistBackupIntegrationSuite(t *testing.T) {
func (suite *AssistBackupIntegrationSuite) SetupSuite() { func (suite *AssistBackupIntegrationSuite) SetupSuite() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -1525,7 +1527,7 @@ func (suite *AssistBackupIntegrationSuite) SetupSuite() {
suite.acct = tconfig.NewM365Account(t) suite.acct = tconfig.NewM365Account(t)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) { suite.kopiaCloser = func(ctx context.Context) {

View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -39,8 +40,9 @@ func getKopiaHandles(
ctx context.Context, //revive:disable-line:context-as-argument ctx context.Context, //revive:disable-line:context-as-argument
) (*kopia.Wrapper, *kopia.ModelStore) { ) (*kopia.Wrapper, *kopia.ModelStore) {
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
repoNameHash := strTD.NewHashForRepoConfigName()
k := kopia.NewConn(st) k := kopia.NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
kw, err := kopia.NewWrapper(k) kw, err := kopia.NewWrapper(k)

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -238,11 +239,12 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() {
var ( var (
st = storeTD.NewPrefixedS3Storage(t) st = storeTD.NewPrefixedS3Storage(t)
k = kopia.NewConn(st) k = kopia.NewConn(st)
repoNameHash = strTD.NewHashForRepoConfigName()
) )
suite.acct = tconfig.NewM365Account(t) suite.acct = tconfig.NewM365Account(t)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) { suite.kopiaCloser = func(ctx context.Context) {

View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -35,12 +36,13 @@ func (suite *RetentionConfigOpIntegrationSuite) TestRepoRetentionConfig() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st = storeTD.NewPrefixedS3Storage(t) st = storeTD.NewPrefixedS3Storage(t)
k = kopia.NewConn(st) k = kopia.NewConn(st)
repoNameHash = strTD.NewHashForRepoConfigName()
) )
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
kw, err := kopia.NewWrapper(k) kw, err := kopia.NewWrapper(k)

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -124,10 +125,11 @@ func prepNewTestBackupOp(
acct: tconfig.NewM365Account(t), acct: tconfig.NewM365Account(t),
st: storeTD.NewPrefixedS3Storage(t), st: storeTD.NewPrefixedS3Storage(t),
} }
repoNameHash := strTD.NewHashForRepoConfigName()
k := kopia.NewConn(bod.st) k := kopia.NewConn(bod.st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
defer func() { defer func() {

View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia"
@ -82,9 +83,10 @@ func prepNewTestRestoreOp(
st: backupStore, st: backupStore,
} }
k = kopia.NewConn(rod.st) k = kopia.NewConn(rod.st)
repoNameHash = strTD.NewHashForRepoConfigName()
) )
err := k.Connect(ctx, repository.Options{}) err := k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// kopiaRef comes with a count of 1 and Wrapper bumps it again // kopiaRef comes with a count of 1 and Wrapper bumps it again

View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -36,6 +37,7 @@ func TestStreamStoreIntgSuite(t *testing.T) {
func (suite *StreamStoreIntgSuite) SetupSubTest() { func (suite *StreamStoreIntgSuite) SetupSubTest() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -44,7 +46,7 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() {
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
k := kopia.NewConn(st) k := kopia.NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.kcloser = func() { k.Close(ctx) } suite.kcloser = func() { k.Close(ctx) }

View File

@ -17,6 +17,7 @@ const (
// storage parsing errors // storage parsing errors
var ( var (
errMissingRequired = clues.New("missing required storage configuration") errMissingRequired = clues.New("missing required storage configuration")
errInvalidProvider = clues.New("unsupported account provider")
) )
const ( const (
@ -38,6 +39,7 @@ type providerIDer interface {
common.StringConfigurer common.StringConfigurer
providerID(accountProvider) string providerID(accountProvider) string
configHash() (string, error)
} }
// NewAccount aggregates all the supplied configurations into a single configuration // NewAccount aggregates all the supplied configurations into a single configuration
@ -88,3 +90,17 @@ func (a Account) ID() string {
return a.Config[a.Provider.String()+"-tenant-id"] return a.Config[a.Provider.String()+"-tenant-id"]
} }
func (a Account) GetAccountConfigHash() (string, error) {
switch a.Provider {
case ProviderM365:
m365, err := a.M365Config()
if err != nil {
return "", clues.Stack(err)
}
return m365.configHash()
}
return "", errInvalidProvider.With("provider", a.Provider)
}

View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
) )
@ -22,6 +23,10 @@ func (c testConfig) StringConfig() (map[string]string, error) {
return map[string]string{"expect": c.expect}, c.err return map[string]string{"expect": c.expect}, c.err
} }
func (c testConfig) configHash() (string, error) {
return "hashed-config", c.err
}
type AccountSuite struct { type AccountSuite struct {
suite.Suite suite.Suite
} }
@ -63,3 +68,57 @@ func (suite *AccountSuite) TestNewAccount() {
}) })
} }
} }
func (suite *AccountSuite) TestGetAccountConfigHash() {
tests := []struct {
name string
provider accountProvider
config any
}{
{
name: "valid account",
provider: ProviderM365,
config: getTestM365Config("1234", "5678"),
},
{
name: "invalid account",
provider: ProviderUnknown,
config: testConfig{"configVal", "", nil},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
if test.provider == ProviderUnknown {
s, err := NewAccount(test.provider, test.config.(testConfig))
require.NoError(t, err)
_, err = s.GetAccountConfigHash()
require.Error(t, err)
}
if test.provider == ProviderM365 {
_, ok := test.config.(providerIDer)
require.True(t, ok)
s, err := NewAccount(test.provider, test.config.(M365Config))
require.NoError(t, err)
hash, err := s.GetAccountConfigHash()
require.NoError(t, err)
assert.True(t, len(hash) > 0)
}
})
}
}
func getTestM365Config(clientID, tenantID string) M365Config {
c := M365Config{}
c.AzureClientID = clientID
c.AzureClientSecret = "super secret"
c.AzureTenantID = tenantID
return c
}

View File

@ -1,8 +1,13 @@
package account package account
import ( import (
"encoding/json"
"reflect"
"slices"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
) )
@ -11,6 +16,8 @@ const (
AzureTenantID = "AZURE_TENANT_ID" AzureTenantID = "AZURE_TENANT_ID"
) )
var excludedM365ConfigFieldsForHashing = []string{"AzureClientSecret"}
type M365Config struct { type M365Config struct {
credentials.M365 // requires: ClientID, ClientSecret credentials.M365 // requires: ClientID, ClientSecret
AzureTenantID string AzureTenantID string
@ -45,6 +52,17 @@ func (c M365Config) providerID(ap accountProvider) string {
return "" return ""
} }
func (c M365Config) configHash() (string, error) {
filteredM365Config := createFilteredM365ConfigForHashing(c)
b, err := json.Marshal(filteredM365Config)
if err != nil {
return "", clues.Stack(err)
}
return str.GenerateHash(b), nil
}
// M365Config retrieves the M365Config details from the Account config. // M365Config retrieves the M365Config details from the Account config.
func (a Account) M365Config() (M365Config, error) { func (a Account) M365Config() (M365Config, error) {
c := M365Config{} c := M365Config{}
@ -57,6 +75,20 @@ func (a Account) M365Config() (M365Config, error) {
return c, c.validate() return c, c.validate()
} }
func createFilteredM365ConfigForHashing(source M365Config) map[string]any {
filteredM365Config := make(map[string]any)
sourceValue := reflect.ValueOf(source)
for i := 0; i < sourceValue.NumField(); i++ {
fieldName := sourceValue.Type().Field(i).Name
if !slices.Contains(excludedM365ConfigFieldsForHashing, fieldName) {
filteredM365Config[fieldName] = sourceValue.Field(i).Interface()
}
}
return filteredM365Config
}
func (c M365Config) validate() error { func (c M365Config) validate() error {
check := map[string]string{ check := map[string]string{
credentials.AzureClientID: c.AzureClientID, credentials.AzureClientID: c.AzureClientID,

View File

@ -2,6 +2,7 @@ package repository
import ( import (
"context" "context"
"fmt"
"time" "time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -211,12 +212,17 @@ func (r *repository) UpdatePassword(ctx context.Context, password string) (err e
progressBar := observe.MessageWithCompletion(ctx, "Connecting to repository") progressBar := observe.MessageWithCompletion(ctx, "Connecting to repository")
defer close(progressBar) defer close(progressBar)
repoNameHash, err := r.GenerateHashForRepositoryConfigFileName()
if err != nil {
return clues.Wrap(err, "generating repo config hash")
}
kopiaRef := kopia.NewConn(r.Storage) kopiaRef := kopia.NewConn(r.Storage)
if err := kopiaRef.Connect(ctx, r.Opts.Repo); err != nil { if err := kopiaRef.Connect(ctx, r.Opts.Repo, repoNameHash); err != nil {
return clues.Wrap(err, "connecting kopia client") return clues.Wrap(err, "connecting kopia client")
} }
err = kopiaRef.UpdatePassword(ctx, password, r.Opts.Repo) err = kopiaRef.UpdatePassword(ctx, password, r.Opts.Repo, repoNameHash)
if err != nil { if err != nil {
return clues.Wrap(err, "updating on kopia") return clues.Wrap(err, "updating on kopia")
} }
@ -294,9 +300,14 @@ func (r *repository) setupKopia(
) error { ) error {
var err error var err error
repoHashName, err := r.GenerateHashForRepositoryConfigFileName()
if err != nil {
return clues.Wrap(err, "generating repo config hash")
}
kopiaRef := kopia.NewConn(r.Storage) kopiaRef := kopia.NewConn(r.Storage)
if isInitialize { if isInitialize {
if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts); err != nil { if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts, repoHashName); err != nil {
// Replace common internal errors so that SDK users can check results with errors.Is() // Replace common internal errors so that SDK users can check results with errors.Is()
if errors.Is(err, kopia.ErrorRepoAlreadyExists) { if errors.Is(err, kopia.ErrorRepoAlreadyExists) {
return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx)
@ -305,7 +316,7 @@ func (r *repository) setupKopia(
return clues.Wrap(err, "initializing kopia") return clues.Wrap(err, "initializing kopia")
} }
} else { } else {
if err := kopiaRef.Connect(ctx, r.Opts.Repo); err != nil { if err := kopiaRef.Connect(ctx, r.Opts.Repo, repoHashName); err != nil {
return clues.Wrap(err, "connecting kopia client") return clues.Wrap(err, "connecting kopia client")
} }
} }
@ -335,6 +346,20 @@ func (r *repository) setupKopia(
return nil return nil
} }
func (r repository) GenerateHashForRepositoryConfigFileName() (string, error) {
accountHash, err := r.Account.GetAccountConfigHash()
if err != nil {
return "", clues.Wrap(err, "fetch account config hash")
}
storageHash, err := r.Storage.GetStorageConfigHash()
if err != nil {
return "", clues.Wrap(err, "fetch storage config hash")
}
return fmt.Sprintf("%s-%s", accountHash, storageHash), nil
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Repository ID Model // Repository ID Model
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/model"
@ -698,6 +699,7 @@ func TestRepositoryModelIntgSuite(t *testing.T) {
func (suite *RepositoryModelIntgSuite) SetupSuite() { func (suite *RepositoryModelIntgSuite) SetupSuite() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -710,10 +712,10 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() {
require.NotNil(t, k) require.NotNil(t, k)
err = k.Initialize(ctx, rep.Options{}, rep.Retention{}) err = k.Initialize(ctx, rep.Options{}, rep.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, rep.Options{}) err = k.Connect(ctx, rep.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) { suite.kopiaCloser = func(ctx context.Context) {
@ -752,6 +754,7 @@ func (suite *RepositoryModelIntgSuite) TearDownSuite() {
func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() {
t := suite.T() t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -761,10 +764,10 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() {
k = kopia.NewConn(s) k = kopia.NewConn(s)
) )
err := k.Initialize(ctx, rep.Options{}, rep.Retention{}) err := k.Initialize(ctx, rep.Options{}, rep.Retention{}, repoNameHash)
require.NoError(t, err, "initializing repo: %v", clues.ToCore(err)) require.NoError(t, err, "initializing repo: %v", clues.ToCore(err))
err = k.Connect(ctx, rep.Options{}) err = k.Connect(ctx, rep.Options{}, repoNameHash)
require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err)) require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err))
defer k.Close(ctx) defer k.Close(ctx)

View File

@ -1,6 +1,9 @@
package storage package storage
import ( import (
"encoding/json"
"reflect"
"slices"
"strings" "strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -10,6 +13,9 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
// nothing to exclude, for parity
var excludedFileSystemConfigFieldsForHashing = []string{}
const ( const (
FilesystemPath = "path" FilesystemPath = "path"
) )
@ -58,6 +64,31 @@ func (c *FilesystemConfig) fsConfigsFromStore(g Getter) {
c.Path = cast.ToString(g.Get(FilesystemPath)) c.Path = cast.ToString(g.Get(FilesystemPath))
} }
func (c FilesystemConfig) configHash() (string, error) {
filteredFileSystemConfig := createFilteredFileSystemConfigForHashing(c)
b, err := json.Marshal(filteredFileSystemConfig)
if err != nil {
return "", clues.Stack(err)
}
return str.GenerateHash(b), nil
}
func createFilteredFileSystemConfigForHashing(source FilesystemConfig) map[string]any {
filteredFileSystemConfig := make(map[string]any)
sourceValue := reflect.ValueOf(source)
for i := 0; i < sourceValue.NumField(); i++ {
fieldName := sourceValue.Type().Field(i).Name
if !slices.Contains(excludedFileSystemConfigFieldsForHashing, fieldName) {
filteredFileSystemConfig[fieldName] = sourceValue.Field(i).Interface()
}
}
return filteredFileSystemConfig
}
// TODO(pandeyabs): Remove this. It's not adding any value. // TODO(pandeyabs): Remove this. It's not adding any value.
func fsOverrides(in map[string]string) map[string]string { func fsOverrides(in map[string]string) map[string]string {
return map[string]string{ return map[string]string{

View File

@ -1,7 +1,10 @@
package storage package storage
import ( import (
"encoding/json"
"os" "os"
"reflect"
"slices"
"strconv" "strconv"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -21,6 +24,14 @@ type S3Config struct {
DoNotVerifyTLS bool DoNotVerifyTLS bool
} }
var excludedS3ConfigFieldsForHashing = []string{
"DoNotUseTLS",
"DoNotVerifyTLS",
"AccessKey",
"SecretKey",
"SessionToken",
}
// config key consts // config key consts
const ( const (
keyS3AccessKey = "s3_access_key" keyS3AccessKey = "s3_access_key"
@ -129,6 +140,31 @@ func (c S3Config) validate() error {
return nil return nil
} }
func (c S3Config) configHash() (string, error) {
filteredS3Config := createFilteredS3ConfigForHashing(c)
b, err := json.Marshal(filteredS3Config)
if err != nil {
return "", clues.Stack(err)
}
return str.GenerateHash(b), nil
}
func createFilteredS3ConfigForHashing(source S3Config) map[string]any {
filteredS3Config := make(map[string]any)
sourceValue := reflect.ValueOf(source)
for i := 0; i < sourceValue.NumField(); i++ {
fieldName := sourceValue.Type().Field(i).Name
if !slices.Contains(excludedS3ConfigFieldsForHashing, fieldName) {
filteredS3Config[fieldName] = sourceValue.Field(i).Interface()
}
}
return filteredS3Config
}
func s3Overrides(in map[string]string) map[string]string { func s3Overrides(in map[string]string) map[string]string {
return map[string]string{ return map[string]string{
Bucket: in[Bucket], Bucket: in[Bucket],

View File

@ -35,6 +35,7 @@ const (
// storage parsing errors // storage parsing errors
var ( var (
errMissingRequired = clues.New("missing required storage configuration") errMissingRequired = clues.New("missing required storage configuration")
errInvalidProvider = clues.New("unsupported storage provider")
) )
// Storage defines a storage provider, along with any configuration // Storage defines a storage provider, along with any configuration
@ -106,7 +107,29 @@ func (s Storage) StorageConfig() (Configurer, error) {
return buildFilesystemConfigFromMap(s.Config) return buildFilesystemConfigFromMap(s.Config)
} }
return nil, clues.New("unsupported storage provider: [" + s.Provider.String() + "]") return nil, errInvalidProvider.With("provider", s.Provider)
}
func (s Storage) GetStorageConfigHash() (string, error) {
switch s.Provider {
case ProviderS3:
s3Cnf, err := s.ToS3Config()
if err != nil {
return "", err
}
return s3Cnf.configHash()
case ProviderFilesystem:
fsCnf, err := s.ToFilesystemConfig()
if err != nil {
return "", err
}
return fsCnf.configHash()
}
return "", errInvalidProvider.With("provider", s.Provider)
} }
func NewStorageConfig(provider ProviderType) (Configurer, error) { func NewStorageConfig(provider ProviderType) (Configurer, error) {
@ -117,7 +140,7 @@ func NewStorageConfig(provider ProviderType) (Configurer, error) {
return &FilesystemConfig{}, nil return &FilesystemConfig{}, nil
} }
return nil, clues.New("unsupported storage provider: [" + provider.String() + "]") return nil, errInvalidProvider.With("provider", provider)
} }
type Getter interface { type Getter interface {
@ -149,6 +172,8 @@ type Configurer interface {
) error ) error
WriteConfigToStorer WriteConfigToStorer
configHash() (string, error)
} }
// mustMatchConfig compares the values of each key to their config file value in store. // mustMatchConfig compares the values of each key to their config file value in store.

View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -63,6 +64,84 @@ func (suite *StorageUnitSuite) TestNewStorage() {
} }
} }
func (suite *StorageUnitSuite) TestGetAccountConfigHash() {
tests := []struct {
name string
provider ProviderType
config any
}{
{
name: "s3 storage",
provider: ProviderS3,
config: getTestS3Config("test-bucket", "https://aws.s3", "test-prefix"),
},
{
name: "filesystem storage",
provider: ProviderFilesystem,
config: getTestFileSystemConfig("test/to/dir"),
},
{
name: "invalid account",
provider: ProviderUnknown,
config: testConfig{"configVal", nil},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
if test.provider == ProviderUnknown {
s, err := NewStorage(test.provider, test.config.(testConfig))
require.NoError(t, err)
_, err = s.GetStorageConfigHash()
require.Error(t, err)
}
if test.provider == ProviderS3 {
_, ok := test.config.(Configurer)
require.True(t, ok)
s3Cnf := test.config.(*S3Config)
s, err := NewStorage(test.provider, s3Cnf)
require.NoError(t, err)
hash, err := s.GetStorageConfigHash()
require.NoError(t, err)
assert.True(t, len(hash) > 0)
}
if test.provider == ProviderFilesystem {
_, ok := test.config.(Configurer)
require.True(t, ok)
fsCnf := test.config.(*FilesystemConfig)
s, err := NewStorage(test.provider, fsCnf)
require.NoError(t, err)
hash, err := s.GetStorageConfigHash()
require.NoError(t, err)
assert.True(t, len(hash) > 0)
}
})
}
}
func getTestS3Config(bucket, endpoint, prefix string) *S3Config {
return &S3Config{
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
}
}
func getTestFileSystemConfig(path string) *FilesystemConfig {
return &FilesystemConfig{
Path: path,
}
}
type testGetter struct { type testGetter struct {
storeMap map[string]string storeMap map[string]string
} }