larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

generate repository config name based on provider specific hash (#4639)

Browse Source 
Enable user/client to be able to perform multiple backups in a concurrent manner irrespective of storage provider (s3/filesystem) for different tenants (anything that differentiates one account from another).

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #4443 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Hitesh Pattanayak 2023-11-11 14:57:06 +05:30 committed by GitHub
parent 0b997e6176
commit 40c7476e24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 484 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/internal/common/str/str.go
View File

@ -1,7 +1,9 @@
package str
import (
"encoding/hex"
"fmt"
"hash/crc32"
"strconv"
"github.com/alcionai/clues"
@ -90,3 +92,12 @@ func SliceToMap(ss []string) map[string]struct{} {
return m
}
func GenerateHash(input []byte) string {
crc32Hash := crc32.NewIEEE()
crc32Hash.Write(input)
checksum := crc32Hash.Sum(nil)
hashString := hex.EncodeToString(checksum)
return hashString
}

67
src/internal/common/str/str_test.go
View File

@ -1,9 +1,11 @@
package str
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ---------------------------------------------------------------------------
@ -51,3 +53,68 @@ func TestPreview(t *testing.T) {
})
}
}
// Test GenerateHash
func TestGenerateHash(t *testing.T) {
type testStruct struct {
Text string
Number int
Status bool
}
table := []struct {
name string
input1 any
input2 any
sameCheck bool
}{
{
name: "check if same hash is generated for same string input",
input1: "test data",
sameCheck: true,
},
{
name: "check if same hash is generated for same struct input",
input1: testStruct{Text: "test text", Number: 1, Status: true},
sameCheck: true,
},
{
name: "check if different hash is generated for different string input",
input1: "test data",
input2: "test data 2",
sameCheck: false,
},
{
name: "check if different hash is generated for different struct input",
input1: testStruct{Text: "test text", Number: 1, Status: true},
input2: testStruct{Text: "test text 2", Number: 2, Status: false},
sameCheck: false,
},
}
for _, test := range table {
var input1Bytes []byte
var err error
var hash1 string
input1Bytes, err = json.Marshal(test.input1)
require.NoError(t, err)
hash1 = GenerateHash(input1Bytes)
if test.sameCheck {
hash2 := GenerateHash(input1Bytes)
assert.Equal(t, hash1, hash2)
} else {
input2Bytes, err := json.Marshal(test.input2)
require.NoError(t, err)
hash2 := GenerateHash(input2Bytes)
assert.NotEqual(t, hash1, hash2)
}
}
}

9
src/internal/common/str/testdata/str.go vendored Normal file
View File

@ -0,0 +1,9 @@
package testdata
import "github.com/google/uuid"
const hashLength = 7
func NewHashForRepoConfigName() string {
return uuid.NewString()[:hashLength]
}

20
src/internal/kopia/conn.go
View File

@ -2,6 +2,7 @@ package kopia
import (
"context"
"fmt"
"path/filepath"
"sync"
"time"
@ -28,7 +29,7 @@ import (
const (
defaultKopiaConfigDir = "/tmp/"
defaultKopiaConfigFile = "repository.config"
kopiaConfigFileTemplate = "repository-%s.config"
defaultCompressor = "zstd-better-compression"
// Interval of 0 disables scheduling.
defaultSchedulingInterval = time.Second * 0
@ -95,6 +96,7 @@ func (w *conn) Initialize(
ctx context.Context,
opts repository.Options,
retentionOpts repository.Retention,
repoNameHash string,
) error {
bst, err := blobStoreByProvider(ctx, opts, w.storage)
if err != nil {
@ -135,6 +137,7 @@ func (w *conn) Initialize(
ctx,
opts,
cfg.KopiaCfgDir,
repoNameHash,
bst,
cfg.CorsoPassphrase,
defaultCompressor)
@ -152,7 +155,7 @@ func (w *conn) Initialize(
return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil()
}
func (w *conn) Connect(ctx context.Context, opts repository.Options) error {
func (w *conn) Connect(ctx context.Context, opts repository.Options, repoNameHash string) error {
bst, err := blobStoreByProvider(ctx, opts, w.storage)
if err != nil {
return clues.Wrap(err, "initializing storage")
@ -168,6 +171,7 @@ func (w *conn) Connect(ctx context.Context, opts repository.Options) error {
ctx,
opts,
cfg.KopiaCfgDir,
repoNameHash,
bst,
cfg.CorsoPassphrase,
defaultCompressor)
@ -177,6 +181,7 @@ func (w *conn) commonConnect(
ctx context.Context,
opts repository.Options,
configDir string,
repoNameHash string,
bst blob.Storage,
password, compressor string,
) error {
@ -196,7 +201,7 @@ func (w *conn) commonConnect(
configDir = defaultKopiaConfigDir
}
cfgFile := filepath.Join(configDir, defaultKopiaConfigFile)
cfgFile := filepath.Join(configDir, fmt.Sprintf(kopiaConfigFileTemplate, repoNameHash))
// todo - issue #75: nil here should be storage.ConnectOptions()
if err := repo.Connect(
@ -579,13 +584,18 @@ func (w *conn) SnapshotRoot(man *snapshot.Manifest) (fs.Entry, error) {
return snapshotfs.SnapshotRoot(w.Repository, man)
}
func (w *conn) UpdatePassword(ctx context.Context, password string, opts repository.Options) error {
func (w *conn) UpdatePassword(
ctx context.Context,
password string,
opts repository.Options,
repoNameHash string,
) error {
if len(password) <= 0 {
return clues.New("empty password provided")
}
kopiaRef := NewConn(w.storage)
if err := kopiaRef.Connect(ctx, opts); err != nil {
if err := kopiaRef.Connect(ctx, opts, repoNameHash); err != nil {
return clues.Wrap(err, "connecting kopia client")
}

37
src/internal/kopia/conn_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/storage"
@ -27,9 +28,10 @@ func openLocalKopiaRepo(
ctx context.Context, //revive:disable-line:context-as-argument
) (*conn, error) {
st := storeTD.NewFilesystemStorage(t)
repoNameHash := strTD.NewHashForRepoConfigName()
k := NewConn(st)
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil {
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash); err != nil {
return nil, err
}
@ -41,9 +43,10 @@ func openKopiaRepo(
ctx context.Context, //revive:disable-line:context-as-argument
) (*conn, error) {
st := storeTD.NewPrefixedS3Storage(t)
repoNameHash := strTD.NewHashForRepoConfigName()
k := NewConn(st)
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil {
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash); err != nil {
return nil, err
}
@ -91,6 +94,7 @@ func TestWrapperIntegrationSuite(t *testing.T) {
func (suite *WrapperIntegrationSuite) TestRepoExistsError() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -98,19 +102,20 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() {
st := storeTD.NewFilesystemStorage(t)
k := NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
err = k.Close(ctx)
require.NoError(t, err, clues.ToCore(err))
err = k.Initialize(ctx, repository.Options{}, repository.Retention{})
err = k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
assert.Error(t, err, clues.ToCore(err))
assert.ErrorIs(t, err, ErrorRepoAlreadyExists)
}
func (suite *WrapperIntegrationSuite) TestBadProviderErrors() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -119,12 +124,13 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() {
st.Provider = storage.ProviderUnknown
k := NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
assert.Error(t, err, clues.ToCore(err))
}
func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -132,7 +138,7 @@ func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() {
st := storeTD.NewFilesystemStorage(t)
k := NewConn(st)
err := k.Connect(ctx, repository.Options{})
err := k.Connect(ctx, repository.Options{}, repoNameHash)
assert.Error(t, err, clues.ToCore(err))
}
@ -282,6 +288,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect
newRetentionDaily := policy.OptionalInt(42)
newRetention := policy.RetentionPolicy{KeepDaily: &newRetentionDaily}
newSchedInterval := time.Second * 42
repoNameHash := strTD.NewHashForRepoConfigName()
table := []struct {
name string
@ -376,7 +383,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect
err = k.Close(ctx)
require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, repository.Options{})
err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
defer func() {
@ -393,6 +400,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect
func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -404,7 +412,7 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() {
require.NoError(t, err, clues.ToCore(err))
// Re-open with Connect.
err = k.Connect(ctx, repository.Options{})
err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
err = k.Close(ctx)
@ -413,6 +421,7 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() {
func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -425,7 +434,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
st := storeTD.NewFilesystemStorage(t)
k := NewConn(st)
err := k.Initialize(ctx, opts, repository.Retention{})
err := k.Initialize(ctx, opts, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
kopiaOpts := k.ClientOptions()
@ -439,7 +448,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
opts.User = "hello"
opts.Host = "world"
err = k.Connect(ctx, opts)
err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
kopiaOpts = k.ClientOptions()
@ -453,7 +462,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
opts.User = ""
opts.Host = ""
err = k.Connect(ctx, opts)
err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
kopiaOpts = k.ClientOptions()
@ -485,6 +494,7 @@ func TestConnRetentionIntegrationSuite(t *testing.T) {
// from the default values that kopia uses.
func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -492,7 +502,7 @@ func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
st1 := storeTD.NewPrefixedS3Storage(t)
k1 := NewConn(st1)
err := k1.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err))
st2 := storeTD.NewPrefixedS3Storage(t)
@ -505,7 +515,8 @@ func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
Mode: ptr.To(repository.GovernanceRetention),
Duration: ptr.To(time.Hour * 48),
Extend: ptr.To(true),
})
},
repoNameHash)
require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err))
dr1, ok := k1.Repository.(repo.DirectRepository)

7
src/internal/kopia/model_store_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/tester"
@ -858,8 +859,9 @@ func openConnAndModelStore(
) (*conn, *ModelStore) {
st := storeTD.NewFilesystemStorage(t)
c := NewConn(st)
repoNameHash := strTD.NewHashForRepoConfigName()
err := c.Initialize(ctx, repository.Options{}, repository.Retention{})
err := c.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
defer func() {
@ -878,7 +880,8 @@ func reconnectToModelStore(
ctx context.Context, //revive:disable-line:context-as-argument
c *conn,
) *ModelStore {
err := c.Connect(ctx, repository.Options{})
repoNameHash := strTD.NewHashForRepoConfigName()
err := c.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
defer func() {

13
src/internal/kopia/wrapper_test.go
View File

@ -23,6 +23,7 @@ import (
pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock"
"github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
@ -202,6 +203,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_FirstRun_NoChanges() {
func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -228,7 +230,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails
Host: "bar",
}
err = k.Connect(ctx, opts)
err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
var notOwnedErr maintenance.NotOwnedError
@ -239,6 +241,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_NoForce_Fails
func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeeds() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -265,7 +268,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed
Host: "bar",
}
err = k.Connect(ctx, opts)
err = k.Connect(ctx, opts, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
mOpts.Force = true
@ -286,6 +289,7 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed
// blobs as there's several of them, but at least this gives us something.
func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnFailure() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -318,7 +322,7 @@ func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnF
k.Close(ctx)
require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, repository.Options{})
err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
defer k.Close(ctx)
@ -375,6 +379,7 @@ func checkRetentionParams(
//revive:disable-next-line:context-as-argument
func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) {
k := w.c
repoNameHash := strTD.NewHashForRepoConfigName()
err := w.Close(ctx)
require.NoError(t, err, "closing wrapper: %v", clues.ToCore(err))
@ -382,7 +387,7 @@ func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) {
err = k.Close(ctx)
require.NoError(t, err, "closing conn: %v", clues.ToCore(err))
err = k.Connect(ctx, repository.Options{})
err = k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, "reconnecting conn: %v", clues.ToCore(err))
w.c = k

4
src/internal/operations/backup_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -1514,6 +1515,7 @@ func TestAssistBackupIntegrationSuite(t *testing.T) {
func (suite *AssistBackupIntegrationSuite) SetupSuite() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -1525,7 +1527,7 @@ func (suite *AssistBackupIntegrationSuite) SetupSuite() {
suite.acct = tconfig.NewM365Account(t)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) {

4
src/internal/operations/maintenance_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -39,8 +40,9 @@ func getKopiaHandles(
ctx context.Context, //revive:disable-line:context-as-argument
) (*kopia.Wrapper, *kopia.ModelStore) {
st := storeTD.NewPrefixedS3Storage(t)
repoNameHash := strTD.NewHashForRepoConfigName()
k := kopia.NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
kw, err := kopia.NewWrapper(k)

4
src/internal/operations/restore_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -238,11 +239,12 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() {
var (
st = storeTD.NewPrefixedS3Storage(t)
k = kopia.NewConn(st)
repoNameHash = strTD.NewHashForRepoConfigName()
)
suite.acct = tconfig.NewM365Account(t)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) {

4
src/internal/operations/retention_config_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/tester"
@ -35,12 +36,13 @@ func (suite *RetentionConfigOpIntegrationSuite) TestRepoRetentionConfig() {
// need to initialize the repository before we can test connecting to it.
st = storeTD.NewPrefixedS3Storage(t)
k = kopia.NewConn(st)
repoNameHash = strTD.NewHashForRepoConfigName()
)
ctx, flush := tester.NewContext(t)
defer flush()
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
kw, err := kopia.NewWrapper(k)

4
src/internal/operations/test/helper_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -124,10 +125,11 @@ func prepNewTestBackupOp(
acct: tconfig.NewM365Account(t),
st: storeTD.NewPrefixedS3Storage(t),
}
repoNameHash := strTD.NewHashForRepoConfigName()
k := kopia.NewConn(bod.st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
defer func() {

4
src/internal/operations/test/restore_helper_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/common/idname"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia"
@ -82,9 +83,10 @@ func prepNewTestRestoreOp(
st: backupStore,
}
k = kopia.NewConn(rod.st)
repoNameHash = strTD.NewHashForRepoConfigName()
)
err := k.Connect(ctx, repository.Options{})
err := k.Connect(ctx, repository.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
// kopiaRef comes with a count of 1 and Wrapper bumps it again

4
src/internal/streamstore/collectables_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/tester"
@ -36,6 +37,7 @@ func TestStreamStoreIntgSuite(t *testing.T) {
func (suite *StreamStoreIntgSuite) SetupSubTest() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -44,7 +46,7 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() {
st := storeTD.NewPrefixedS3Storage(t)
k := kopia.NewConn(st)
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
suite.kcloser = func() { k.Close(ctx) }

16
src/pkg/account/account.go
View File

@ -17,6 +17,7 @@ const (
// storage parsing errors
var (
errMissingRequired = clues.New("missing required storage configuration")
errInvalidProvider = clues.New("unsupported account provider")
)
const (
@ -38,6 +39,7 @@ type providerIDer interface {
common.StringConfigurer
providerID(accountProvider) string
configHash() (string, error)
}
// NewAccount aggregates all the supplied configurations into a single configuration
@ -88,3 +90,17 @@ func (a Account) ID() string {
return a.Config[a.Provider.String()+"-tenant-id"]
}
func (a Account) GetAccountConfigHash() (string, error) {
switch a.Provider {
case ProviderM365:
m365, err := a.M365Config()
if err != nil {
return "", clues.Stack(err)
}
return m365.configHash()
}
return "", errInvalidProvider.With("provider", a.Provider)
}

59
src/pkg/account/account_test.go
View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@ -22,6 +23,10 @@ func (c testConfig) StringConfig() (map[string]string, error) {
return map[string]string{"expect": c.expect}, c.err
}
func (c testConfig) configHash() (string, error) {
return "hashed-config", c.err
}
type AccountSuite struct {
suite.Suite
}
@ -63,3 +68,57 @@ func (suite *AccountSuite) TestNewAccount() {
})
}
}
func (suite *AccountSuite) TestGetAccountConfigHash() {
tests := []struct {
name string
provider accountProvider
config any
}{
{
name: "valid account",
provider: ProviderM365,
config: getTestM365Config("1234", "5678"),
},
{
name: "invalid account",
provider: ProviderUnknown,
config: testConfig{"configVal", "", nil},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
if test.provider == ProviderUnknown {
s, err := NewAccount(test.provider, test.config.(testConfig))
require.NoError(t, err)
_, err = s.GetAccountConfigHash()
require.Error(t, err)
}
if test.provider == ProviderM365 {
_, ok := test.config.(providerIDer)
require.True(t, ok)
s, err := NewAccount(test.provider, test.config.(M365Config))
require.NoError(t, err)
hash, err := s.GetAccountConfigHash()
require.NoError(t, err)
assert.True(t, len(hash) > 0)
}
})
}
}
func getTestM365Config(clientID, tenantID string) M365Config {
c := M365Config{}
c.AzureClientID = clientID
c.AzureClientSecret = "super secret"
c.AzureTenantID = tenantID
return c
}

32
src/pkg/account/m365.go
View File

@ -1,8 +1,13 @@
package account
import (
"encoding/json"
"reflect"
"slices"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials"
)
@ -11,6 +16,8 @@ const (
AzureTenantID = "AZURE_TENANT_ID"
)
var excludedM365ConfigFieldsForHashing = []string{"AzureClientSecret"}
type M365Config struct {
credentials.M365 // requires: ClientID, ClientSecret
AzureTenantID string
@ -45,6 +52,17 @@ func (c M365Config) providerID(ap accountProvider) string {
return ""
}
func (c M365Config) configHash() (string, error) {
filteredM365Config := createFilteredM365ConfigForHashing(c)
b, err := json.Marshal(filteredM365Config)
if err != nil {
return "", clues.Stack(err)
}
return str.GenerateHash(b), nil
}
// M365Config retrieves the M365Config details from the Account config.
func (a Account) M365Config() (M365Config, error) {
c := M365Config{}
@ -57,6 +75,20 @@ func (a Account) M365Config() (M365Config, error) {
return c, c.validate()
}
func createFilteredM365ConfigForHashing(source M365Config) map[string]any {
filteredM365Config := make(map[string]any)
sourceValue := reflect.ValueOf(source)
for i := 0; i < sourceValue.NumField(); i++ {
fieldName := sourceValue.Type().Field(i).Name
if !slices.Contains(excludedM365ConfigFieldsForHashing, fieldName) {
filteredM365Config[fieldName] = sourceValue.Field(i).Interface()
}
}
return filteredM365Config
}
func (c M365Config) validate() error {
check := map[string]string{
credentials.AzureClientID: c.AzureClientID,

33
src/pkg/repository/repository.go
View File

@ -2,6 +2,7 @@ package repository
import (
"context"
"fmt"
"time"
"github.com/alcionai/clues"
@ -211,12 +212,17 @@ func (r *repository) UpdatePassword(ctx context.Context, password string) (err e
progressBar := observe.MessageWithCompletion(ctx, "Connecting to repository")
defer close(progressBar)
repoNameHash, err := r.GenerateHashForRepositoryConfigFileName()
if err != nil {
return clues.Wrap(err, "generating repo config hash")
}
kopiaRef := kopia.NewConn(r.Storage)
if err := kopiaRef.Connect(ctx, r.Opts.Repo); err != nil {
if err := kopiaRef.Connect(ctx, r.Opts.Repo, repoNameHash); err != nil {
return clues.Wrap(err, "connecting kopia client")
}
err = kopiaRef.UpdatePassword(ctx, password, r.Opts.Repo)
err = kopiaRef.UpdatePassword(ctx, password, r.Opts.Repo, repoNameHash)
if err != nil {
return clues.Wrap(err, "updating on kopia")
}
@ -294,9 +300,14 @@ func (r *repository) setupKopia(
) error {
var err error
repoHashName, err := r.GenerateHashForRepositoryConfigFileName()
if err != nil {
return clues.Wrap(err, "generating repo config hash")
}
kopiaRef := kopia.NewConn(r.Storage)
if isInitialize {
if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts); err != nil {
if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts, repoHashName); err != nil {
// Replace common internal errors so that SDK users can check results with errors.Is()
if errors.Is(err, kopia.ErrorRepoAlreadyExists) {
return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx)
@ -305,7 +316,7 @@ func (r *repository) setupKopia(
return clues.Wrap(err, "initializing kopia")
}
} else {
if err := kopiaRef.Connect(ctx, r.Opts.Repo); err != nil {
if err := kopiaRef.Connect(ctx, r.Opts.Repo, repoHashName); err != nil {
return clues.Wrap(err, "connecting kopia client")
}
}
@ -335,6 +346,20 @@ func (r *repository) setupKopia(
return nil
}
func (r repository) GenerateHashForRepositoryConfigFileName() (string, error) {
accountHash, err := r.Account.GetAccountConfigHash()
if err != nil {
return "", clues.Wrap(err, "fetch account config hash")
}
storageHash, err := r.Storage.GetStorageConfigHash()
if err != nil {
return "", clues.Wrap(err, "fetch storage config hash")
}
return fmt.Sprintf("%s-%s", accountHash, storageHash), nil
}
// ---------------------------------------------------------------------------
// Repository ID Model
// ---------------------------------------------------------------------------

11
src/pkg/repository/repository_unexported_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/model"
@ -698,6 +699,7 @@ func TestRepositoryModelIntgSuite(t *testing.T) {
func (suite *RepositoryModelIntgSuite) SetupSuite() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -710,10 +712,10 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() {
require.NotNil(t, k)
err = k.Initialize(ctx, rep.Options{}, rep.Retention{})
err = k.Initialize(ctx, rep.Options{}, rep.Retention{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, rep.Options{})
err = k.Connect(ctx, rep.Options{}, repoNameHash)
require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) {
@ -752,6 +754,7 @@ func (suite *RepositoryModelIntgSuite) TearDownSuite() {
func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() {
t := suite.T()
repoNameHash := strTD.NewHashForRepoConfigName()
ctx, flush := tester.NewContext(t)
defer flush()
@ -761,10 +764,10 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() {
k = kopia.NewConn(s)
)
err := k.Initialize(ctx, rep.Options{}, rep.Retention{})
err := k.Initialize(ctx, rep.Options{}, rep.Retention{}, repoNameHash)
require.NoError(t, err, "initializing repo: %v", clues.ToCore(err))
err = k.Connect(ctx, rep.Options{})
err = k.Connect(ctx, rep.Options{}, repoNameHash)
require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err))
defer k.Close(ctx)

31
src/pkg/storage/filesystem.go
View File

@ -1,6 +1,9 @@
package storage
import (
"encoding/json"
"reflect"
"slices"
"strings"
"github.com/alcionai/clues"
@ -10,6 +13,9 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
// nothing to exclude, for parity
var excludedFileSystemConfigFieldsForHashing = []string{}
const (
FilesystemPath = "path"
)
@ -58,6 +64,31 @@ func (c *FilesystemConfig) fsConfigsFromStore(g Getter) {
c.Path = cast.ToString(g.Get(FilesystemPath))
}
func (c FilesystemConfig) configHash() (string, error) {
filteredFileSystemConfig := createFilteredFileSystemConfigForHashing(c)
b, err := json.Marshal(filteredFileSystemConfig)
if err != nil {
return "", clues.Stack(err)
}
return str.GenerateHash(b), nil
}
func createFilteredFileSystemConfigForHashing(source FilesystemConfig) map[string]any {
filteredFileSystemConfig := make(map[string]any)
sourceValue := reflect.ValueOf(source)
for i := 0; i < sourceValue.NumField(); i++ {
fieldName := sourceValue.Type().Field(i).Name
if !slices.Contains(excludedFileSystemConfigFieldsForHashing, fieldName) {
filteredFileSystemConfig[fieldName] = sourceValue.Field(i).Interface()
}
}
return filteredFileSystemConfig
}
// TODO(pandeyabs): Remove this. It's not adding any value.
func fsOverrides(in map[string]string) map[string]string {
return map[string]string{

36
src/pkg/storage/s3.go
View File

@ -1,7 +1,10 @@
package storage
import (
"encoding/json"
"os"
"reflect"
"slices"
"strconv"
"github.com/alcionai/clues"
@ -21,6 +24,14 @@ type S3Config struct {
DoNotVerifyTLS bool
}
var excludedS3ConfigFieldsForHashing = []string{
"DoNotUseTLS",
"DoNotVerifyTLS",
"AccessKey",
"SecretKey",
"SessionToken",
}
// config key consts
const (
keyS3AccessKey = "s3_access_key"
@ -129,6 +140,31 @@ func (c S3Config) validate() error {
return nil
}
func (c S3Config) configHash() (string, error) {
filteredS3Config := createFilteredS3ConfigForHashing(c)
b, err := json.Marshal(filteredS3Config)
if err != nil {
return "", clues.Stack(err)
}
return str.GenerateHash(b), nil
}
func createFilteredS3ConfigForHashing(source S3Config) map[string]any {
filteredS3Config := make(map[string]any)
sourceValue := reflect.ValueOf(source)
for i := 0; i < sourceValue.NumField(); i++ {
fieldName := sourceValue.Type().Field(i).Name
if !slices.Contains(excludedS3ConfigFieldsForHashing, fieldName) {
filteredS3Config[fieldName] = sourceValue.Field(i).Interface()
}
}
return filteredS3Config
}
func s3Overrides(in map[string]string) map[string]string {
return map[string]string{
Bucket: in[Bucket],

29
src/pkg/storage/storage.go
View File

@ -35,6 +35,7 @@ const (
// storage parsing errors
var (
errMissingRequired = clues.New("missing required storage configuration")
errInvalidProvider = clues.New("unsupported storage provider")
)
// Storage defines a storage provider, along with any configuration
@ -106,7 +107,29 @@ func (s Storage) StorageConfig() (Configurer, error) {
return buildFilesystemConfigFromMap(s.Config)
}
return nil, clues.New("unsupported storage provider: [" + s.Provider.String() + "]")
return nil, errInvalidProvider.With("provider", s.Provider)
}
func (s Storage) GetStorageConfigHash() (string, error) {
switch s.Provider {
case ProviderS3:
s3Cnf, err := s.ToS3Config()
if err != nil {
return "", err
}
return s3Cnf.configHash()
case ProviderFilesystem:
fsCnf, err := s.ToFilesystemConfig()
if err != nil {
return "", err
}
return fsCnf.configHash()
}
return "", errInvalidProvider.With("provider", s.Provider)
}
func NewStorageConfig(provider ProviderType) (Configurer, error) {
@ -117,7 +140,7 @@ func NewStorageConfig(provider ProviderType) (Configurer, error) {
return &FilesystemConfig{}, nil
}
return nil, clues.New("unsupported storage provider: [" + provider.String() + "]")
return nil, errInvalidProvider.With("provider", provider)
}
type Getter interface {
@ -149,6 +172,8 @@ type Configurer interface {
) error
WriteConfigToStorer
configHash() (string, error)
}
// mustMatchConfig compares the values of each key to their config file value in store.

79
src/pkg/storage/storage_test.go
View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
@ -63,6 +64,84 @@ func (suite *StorageUnitSuite) TestNewStorage() {
}
}
func (suite *StorageUnitSuite) TestGetAccountConfigHash() {
tests := []struct {
name string
provider ProviderType
config any
}{
{
name: "s3 storage",
provider: ProviderS3,
config: getTestS3Config("test-bucket", "https://aws.s3", "test-prefix"),
},
{
name: "filesystem storage",
provider: ProviderFilesystem,
config: getTestFileSystemConfig("test/to/dir"),
},
{
name: "invalid account",
provider: ProviderUnknown,
config: testConfig{"configVal", nil},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
if test.provider == ProviderUnknown {
s, err := NewStorage(test.provider, test.config.(testConfig))
require.NoError(t, err)
_, err = s.GetStorageConfigHash()
require.Error(t, err)
}
if test.provider == ProviderS3 {
_, ok := test.config.(Configurer)
require.True(t, ok)
s3Cnf := test.config.(*S3Config)
s, err := NewStorage(test.provider, s3Cnf)
require.NoError(t, err)
hash, err := s.GetStorageConfigHash()
require.NoError(t, err)
assert.True(t, len(hash) > 0)
}
if test.provider == ProviderFilesystem {
_, ok := test.config.(Configurer)
require.True(t, ok)
fsCnf := test.config.(*FilesystemConfig)
s, err := NewStorage(test.provider, fsCnf)
require.NoError(t, err)
hash, err := s.GetStorageConfigHash()
require.NoError(t, err)
assert.True(t, len(hash) > 0)
}
})
}
}
func getTestS3Config(bucket, endpoint, prefix string) *S3Config {
return &S3Config{
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
}
}
func getTestFileSystemConfig(path string) *FilesystemConfig {
return &FilesystemConfig{
Path: path,
}
}
type testGetter struct {
storeMap map[string]string
}