diff --git a/src/cli/config/config.go b/src/cli/config/config.go index 39a34eb9c..7fdc9d6eb 100644 --- a/src/cli/config/config.go +++ b/src/cli/config/config.go @@ -218,6 +218,7 @@ func WriteRepoConfig( // writeRepoConfigWithViper implements WriteRepoConfig, but takes in a viper // struct for testing. +// TODO(pandeyabs): This needs to be made generic. Currently it uses s3config. func writeRepoConfigWithViper( vpr *viper.Viper, s3Config storage.S3Config, diff --git a/src/cli/config/storage.go b/src/cli/config/storage.go index d948e6af3..6961290db 100644 --- a/src/cli/config/storage.go +++ b/src/cli/config/storage.go @@ -79,6 +79,7 @@ func configureStorage( if matchFromConfig { providerType := vpr.GetString(StorageProviderTypeKey) if providerType != storage.ProviderS3.String() { + // TODO: needs change return store, clues.New("unsupported storage provider: " + providerType) } @@ -147,7 +148,7 @@ func configureStorage( // ensure required properties are present if err := requireProps(map[string]string{ - storage.Bucket: s3Cfg.Bucket, + //storage.Bucket: s3Cfg.Bucket, credentials.CorsoPassphrase: corso.CorsoPassphrase, }); err != nil { return storage.Storage{}, err diff --git a/src/cli/flags/repo.go b/src/cli/flags/repo.go index 3ec1605ad..feec94b1a 100644 --- a/src/cli/flags/repo.go +++ b/src/cli/flags/repo.go @@ -9,7 +9,7 @@ const ( AWSAccessKeyFN = "aws-access-key" AWSSecretAccessKeyFN = "aws-secret-access-key" AWSSessionTokenFN = "aws-session-token" - + FilesystemPathFN = "path" // Corso Flags CorsoPassphraseFN = "passphrase" ) @@ -19,6 +19,7 @@ var ( AWSAccessKeyFV string AWSSecretAccessKeyFV string AWSSessionTokenFV string + FilesystemPathFV string CorsoPassphraseFV string ) @@ -46,3 +47,12 @@ func AddCorsoPassphaseFlags(cmd *cobra.Command) { "", "Passphrase to protect encrypted repository contents") } + +// AddFilesystemPathFlag adds the --path flag. +func AddFilesystemPathFlag(cmd *cobra.Command, require bool) { + cmd.Flags().StringVar(&FilesystemPathFV, FilesystemPathFN, "", "filesystem path to repo") + + if require { + cobra.CheckErr(cmd.MarkFlagRequired(FilesystemPathFN)) + } +} diff --git a/src/cli/repo/filesystem.go b/src/cli/repo/filesystem.go new file mode 100644 index 000000000..3e1eb7504 --- /dev/null +++ b/src/cli/repo/filesystem.go @@ -0,0 +1,235 @@ +package repo + +import ( + "strings" + + "github.com/alcionai/clues" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/config" + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/pkg/repository" + "github.com/alcionai/corso/src/pkg/storage" +) + +// // filesystem flags +// const ( +// filesystemPathFN = "path" +// ) + +const ( + fsProviderCommand = "filesystem" + fsProviderCommandUseSuffix = "--path " +) + +const ( + fsProviderCommandInitExamples = `# Create a new Corso repo in local or + network attached storage +corso repo init filesystem --path /mnt/corso-repo + +# Create a new Corso repo in local or network attached storage using a prefix +corso repo init filesystem --path /mnt/corso-repo --prefix my-prefix` + + fsProviderCommandConnectExamples = `# Connect to a Corso repo in local or + network attached storage" +corso repo connect filesystem --path /mnt/corso-repo + +# Connect to a Corso repo in local or network attached storage using a prefix +corso repo connect filesystem --path /mnt/corso-repo --prefix my-prefix` +) + +// called by repo.go to map subcommands to provider-specific handling. +func addFsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case initCommand: + init := fsInitCmd() + flags.AddRetentionConfigFlags(init) + c, fs = utils.AddCommand(cmd, init) + + case connectCommand: + c, fs = utils.AddCommand(cmd, fsConnectCmd()) + } + + c.Use = c.Use + " " + fsProviderCommandUseSuffix + c.SetUsageTemplate(cmd.UsageTemplate()) + + flags.AddAzureCredsFlags(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddFilesystemPathFlag(c, true) + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic and more frequently used flags take precedence. + fs.StringVar(&prefix, prefixFN, "", "Repo prefix.") + + // In general, we don't want to expose this flag to users and have them mistake it + // for a broad-scale idempotency solution. We can un-hide it later the need arises. + fs.BoolVar(&succeedIfExists, "succeed-if-exists", false, "Exit with success if the repo has already been initialized.") + cobra.CheckErr(fs.MarkHidden("succeed-if-exists")) + + return c +} + +// `corso repo init filesystem [...]` +func fsInitCmd() *cobra.Command { + return &cobra.Command{ + Use: fsProviderCommand, + Short: "Initialize a repository in local or network attached storage", + Long: `Bootstraps a new local or network attached storage repository + and connects it to your m365 account.`, + RunE: initFsCmd, + Args: cobra.NoArgs, + Example: fsProviderCommandInitExamples, + } +} + +// initializes a filesystem repo. +func initFsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + // TODO(pandeyabs): Modify this func to accept non s3. + cfg, err := config.GetConfigRepoDetails(ctx, true, false, nil) + if err != nil { + return Only(ctx, err) + } + + opt := utils.ControlWithConfig(cfg) + + retentionOpts, err := utils.MakeRetentionOpts(cmd) + if err != nil { + return Only(ctx, err) + } + + // SendStartCorsoEvent uses distict ID as tenant ID because repoID is still not generated + utils.SendStartCorsoEvent( + ctx, + cfg.Storage, + cfg.Account.ID(), + map[string]any{"command": "init repo"}, + cfg.Account.ID(), + opt) + + // TODO(pandeyabs): Need prefix for filesystem repo. + // s3Cfg, err := cfg.Storage.S3Config() + // if err != nil { + // return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration")) + // } + + _, err = cfg.Storage.FsConfig() + if err != nil { + return Only(ctx, clues.Wrap(err, "Retrieving filesystem configuration")) + } + + cfg.Storage.Provider = storage.ProviderFS + + m365, err := cfg.Account.M365Config() + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) + } + + r, err := repository.Initialize( + ctx, + cfg.Account, + cfg.Storage, + opt, + retentionOpts) + if err != nil { + if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) { + return nil + } + + return Only(ctx, clues.Wrap(err, "Failed to initialize a new filesystem repository")) + } + + defer utils.CloseRepo(ctx, r) + + Infof(ctx, "Initialized a repository at path %s", flags.FilesystemPathFV) + + if err = config.WriteRepoConfig(ctx, storage.S3Config{}, m365, opt.Repo, r.GetID()); err != nil { + return Only(ctx, clues.Wrap(err, "Failed to write repository configuration")) + } + + return nil +} + +// --------------------------------------------------------------------------------------------------------- +// Connect +// --------------------------------------------------------------------------------------------------------- + +// `corso repo connect s3 [...]` +func fsConnectCmd() *cobra.Command { + return &cobra.Command{ + Use: fsProviderCommand, + Short: "Connect to a local repository", + Long: `Ensures a connection to an existing local repository.`, + RunE: connectFsCmd, + Args: cobra.NoArgs, + Example: fsProviderCommandConnectExamples, + } +} + +// connects to an existing s3 repo. +func connectFsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + // s3 values from flags + s3Override := S3Overrides(cmd) + + cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override) + if err != nil { + return Only(ctx, err) + } + + repoID := cfg.RepoID + if len(repoID) == 0 { + repoID = events.RepoIDNotFound + } + + s3Cfg, err := cfg.Storage.S3Config() + if err != nil { + return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration")) + } + + m365, err := cfg.Account.M365Config() + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) + } + + if strings.HasPrefix(s3Cfg.Endpoint, "http://") || strings.HasPrefix(s3Cfg.Endpoint, "https://") { + invalidEndpointErr := "endpoint doesn't support specifying protocol. " + + "pass --disable-tls flag to use http:// instead of default https://" + + return Only(ctx, clues.New(invalidEndpointErr)) + } + + opts := utils.ControlWithConfig(cfg) + + r, err := repository.ConnectAndSendConnectEvent( + ctx, + cfg.Account, + cfg.Storage, + repoID, + opts) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to connect to the S3 repository")) + } + + defer utils.CloseRepo(ctx, r) + + Infof(ctx, "Connected to repository at path %s", flags.FilesystemPathFV) + + if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opts.Repo, r.GetID()); err != nil { + return Only(ctx, clues.Wrap(err, "Failed to write repository configuration")) + } + + return nil +} diff --git a/src/cli/repo/repo.go b/src/cli/repo/repo.go index 378cce16a..b88161e8d 100644 --- a/src/cli/repo/repo.go +++ b/src/cli/repo/repo.go @@ -22,6 +22,7 @@ const ( var repoCommands = []func(cmd *cobra.Command) *cobra.Command{ addS3Commands, + addFsCommands, } // AddCommands attaches all `corso repo * *` commands to the parent. diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index e60201d22..7d1306194 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -96,8 +96,6 @@ func (w *conn) Initialize( opts repository.Options, retentionOpts repository.Retention, ) error { - w.storage.Provider = storage.ProviderFS - bst, err := blobStoreByProvider(ctx, opts, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") diff --git a/src/pkg/storage/filesystem.go b/src/pkg/storage/filesystem.go new file mode 100644 index 000000000..d95321f4f --- /dev/null +++ b/src/pkg/storage/filesystem.go @@ -0,0 +1,32 @@ +package storage + +// config exported name consts +const ( + FilesystemPath = "filesystem_path" +) + +type FsConfig struct { + FilesystemPath string +} + +func (s Storage) FsConfig() (FsConfig, error) { + c := FsConfig{} + + if len(s.Config) > 0 { + c.FilesystemPath = orEmptyString(s.Config[FilesystemPath]) + } + + return c, c.validate() +} + +func (c FsConfig) validate() error { + // check := map[string]string{ + // Bucket: c.Bucket, + // } + // for k, v := range check { + // if len(v) == 0 { + // return clues.Stack(errMissingRequired, clues.New(k)) + // } + // } + return nil +} diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index a332326e8..1f134bc4e 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -70,6 +70,7 @@ func (c S3Config) StringConfig() (map[string]string, error) { } // S3Config retrieves the S3Config details from the Storage config. +// TODO(pandeyabs): Make it take s storage as arg. func (s Storage) S3Config() (S3Config, error) { c := S3Config{} @@ -90,7 +91,7 @@ func (s Storage) S3Config() (S3Config, error) { func (c S3Config) validate() error { check := map[string]string{ - Bucket: c.Bucket, + //Bucket: c.Bucket, } for k, v := range check { if len(v) == 0 {